80 #define DEBUG_TYPE "dagcombine"
82 STATISTIC(NodesCombined ,
"Number of dag nodes combined");
83 STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
84 STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
85 STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
86 STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
88 STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
92 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
96 cl::desc(
"Enable DAG combiner's use of TBAA"));
101 cl::desc(
"Only use DAG-combiner alias analysis in this"
109 cl::desc(
"Bypass the profitability model of load slicing"),
114 cl::desc(
"DAG combiner may split indexing from loads"));
118 cl::desc(
"DAG combiner enable merging multiple stores "
119 "into a wider store"));
123 cl::desc(
"Limit the number of operands to inline for Token Factors"));
127 cl::desc(
"Limit the number of times for the same StoreNode and RootNode "
128 "to bail out in store merging dependence check"));
132 cl::desc(
"DAG cominber enable reducing the width of load/op/store "
137 cl::desc(
"DAG cominber enable load/<replace bytes>/store with "
138 "a narrower store"));
148 bool LegalDAG =
false;
149 bool LegalOperations =
false;
150 bool LegalTypes =
false;
152 bool DisableGenericCombines;
193 void AddUsersToWorklist(
SDNode *
N) {
199 void AddToWorklistWithUsers(
SDNode *
N) {
200 AddUsersToWorklist(
N);
207 void clearAddedDanglingWorklistEntries() {
209 while (!PruningList.
empty()) {
212 recursivelyDeleteUnusedNodes(
N);
216 SDNode *getNextWorklistEntry() {
218 clearAddedDanglingWorklistEntries();
222 while (!
N && !Worklist.
empty()) {
227 bool GoodWorklistEntry = WorklistMap.
erase(
N);
228 (void)GoodWorklistEntry;
229 assert(GoodWorklistEntry &&
230 "Found a worklist entry without a corresponding map entry!");
240 : DAG(
D), TLI(
D.getTargetLoweringInfo()),
241 STI(
D.getSubtarget().getSelectionDAGInfo()),
246 MaximumLegalStoreInBits = 0;
252 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits)
253 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize();
256 void ConsiderForPruning(
SDNode *
N) {
263 void AddToWorklist(
SDNode *
N) {
265 "Deleted Node added to Worklist");
272 ConsiderForPruning(
N);
274 if (WorklistMap.
insert(std::make_pair(
N, Worklist.
size())).second)
282 StoreRootCountMap.
erase(
N);
284 auto It = WorklistMap.
find(
N);
285 if (It == WorklistMap.
end())
289 Worklist[It->second] =
nullptr;
290 WorklistMap.
erase(It);
293 void deleteAndRecombine(
SDNode *
N);
294 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
302 return CombineTo(
N, &Res, 1, AddTo);
309 return CombineTo(
N, To, 2, AddTo);
315 unsigned MaximumLegalStoreInBits;
321 unsigned BitWidth =
Op.getScalarValueSizeInBits();
333 AddToWorklist(
Op.getNode());
335 CommitTargetLoweringOpt(TLO);
342 bool SimplifyDemandedVectorElts(
SDValue Op) {
344 if (
Op.getValueType().isScalableVector())
347 unsigned NumElts =
Op.getValueType().getVectorNumElements();
349 return SimplifyDemandedVectorElts(
Op, DemandedElts);
353 const APInt &DemandedElts,
354 bool AssumeSingleUse =
false);
355 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
356 bool AssumeSingleUse =
false);
358 bool CombineToPreIndexedLoadStore(
SDNode *
N);
359 bool CombineToPostIndexedLoadStore(
SDNode *
N);
525 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
543 bool NotExtCompare =
false);
544 SDValue convertSelectOfFPConstantsToLoadOffset(
555 const SDLoc &
DL,
bool foldBooleans);
559 SDValue &CC,
bool MatchStrict =
false)
const;
560 bool isOneUseSetCC(
SDValue N)
const;
583 bool DemandHighBits =
true);
587 unsigned PosOpcode,
unsigned NegOpcode,
591 unsigned PosOpcode,
unsigned NegOpcode,
606 SDValue VecIn2,
unsigned LeftIdx,
641 int64_t OffsetFromBase;
644 : MemNode(
N), OffsetFromBase(
Offset) {}
649 StoreSource getStoreSource(
SDValue StoreVal) {
656 return StoreSource::Extract;
668 bool isMulAddWithConstProfitable(
SDNode *MulNode,
676 EVT LoadResultTy,
EVT &ExtVT);
681 EVT &MemVT,
unsigned ShAmt = 0);
689 bool BackwardsPropagateMask(
SDNode *
N);
701 EVT MemVT,
unsigned NumStores,
702 bool IsConstantSrc,
bool UseVector,
716 bool checkMergeStoreCandidatesForDependencies(
724 int64_t ElementSizeBytes)
const;
729 unsigned NumConsecutiveStores,
730 EVT MemVT,
SDNode *Root,
bool AllowVectors);
737 unsigned NumConsecutiveStores,
EVT MemVT,
743 unsigned NumConsecutiveStores,
EVT MemVT,
744 SDNode *Root,
bool AllowVectors,
745 bool IsNonTemporalStore,
bool IsNonTemporalLoad);
764 bool hasOperation(
unsigned Opcode,
EVT VT) {
776 EVT getShiftAmountTy(
EVT LHSTy) {
783 bool isTypeLegal(
const EVT &VT) {
784 if (!LegalTypes)
return true;
789 EVT getSetCCResultType(
EVT VT)
const {
804 explicit WorklistRemover(DAGCombiner &dc)
808 DC.removeFromWorklist(
N);
816 explicit WorklistInserter(DAGCombiner &dc)
821 void NodeInserted(
SDNode *
N)
override {
DC.ConsiderForPruning(
N); }
831 ((DAGCombiner*)
DC)->AddToWorklist(
N);
836 return ((DAGCombiner*)
DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
841 return ((DAGCombiner*)
DC)->CombineTo(
N, Res, AddTo);
846 return ((DAGCombiner*)
DC)->CombineTo(
N, Res0, Res1, AddTo);
851 return ((DAGCombiner*)
DC)->recursivelyDeleteUnusedNodes(
N);
856 return ((DAGCombiner*)
DC)->CommitTargetLoweringOpt(TLO);
863 void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
872 if (
Op->hasOneUse() ||
Op->getNumValues() > 1)
873 AddToWorklist(
Op.getNode());
893 SDValue &CC,
bool MatchStrict)
const {
895 LHS =
N.getOperand(0);
896 RHS =
N.getOperand(1);
897 CC =
N.getOperand(2);
904 LHS =
N.getOperand(1);
905 RHS =
N.getOperand(2);
906 CC =
N.getOperand(3);
919 LHS =
N.getOperand(0);
920 RHS =
N.getOperand(1);
921 CC =
N.getOperand(4);
928 bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
930 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N.getNode()->hasOneUse())
939 uint64_t MaskForTy = 0ULL;
945 MaskForTy = 0xFFFFULL;
948 MaskForTy = 0xFFFFFFFFULL;
967 return !(Const->isOpaque() && NoOpaques);
970 unsigned BitWidth =
N.getScalarValueSizeInBits();
975 if (!Const || Const->getAPIntValue().getBitWidth() !=
BitWidth ||
976 (Const->isOpaque() && NoOpaques))
995 !cast<ConstantSDNode>(
LD->getOperand(2))->isOpaque());
998 bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1014 auto *C1 = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
1015 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1019 const APInt &C1APIntVal = C1->getAPIntValue();
1020 const APInt &C2APIntVal = C2->getAPIntValue();
1024 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1027 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1030 auto LoadStore = dyn_cast<MemSDNode>(Node);
1037 AM.
BaseOffs = C2APIntVal.getSExtValue();
1039 unsigned AS =
LoadStore->getAddressSpace();
1056 SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &
DL,
1094 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N0, N1))
1096 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N1, N0))
1103 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1107 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1108 for (
unsigned i = 0,
e = NumTo; i !=
e; ++i)
1109 assert((!To[i].getNode() ||
1110 N->getValueType(i) == To[i].getValueType()) &&
1111 "Cannot combine value to value of different type!");
1113 WorklistRemover DeadNodes(*
this);
1117 for (
unsigned i = 0,
e = NumTo; i !=
e; ++i) {
1118 if (To[i].getNode()) {
1119 AddToWorklist(To[i].getNode());
1120 AddUsersToWorklist(To[i].getNode());
1129 deleteAndRecombine(
N);
1143 WorklistRemover DeadNodes(*
this);
1159 const APInt &DemandedElts,
1160 bool AssumeSingleUse) {
1168 AddToWorklist(
Op.getNode());
1170 CommitTargetLoweringOpt(TLO);
1177 bool DAGCombiner::SimplifyDemandedVectorElts(
SDValue Op,
1178 const APInt &DemandedElts,
1179 bool AssumeSingleUse) {
1181 APInt KnownUndef, KnownZero;
1183 TLO, 0, AssumeSingleUse))
1187 AddToWorklist(
Op.getNode());
1189 CommitTargetLoweringOpt(TLO);
1193 void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *
Load,
SDNode *ExtLoad) {
1195 EVT VT =
Load->getValueType(0);
1200 WorklistRemover DeadNodes(*
this);
1203 deleteAndRecombine(
Load);
1204 AddToWorklist(Trunc.
getNode());
1212 EVT MemVT =
LD->getMemoryVT();
1214 :
LD->getExtensionType();
1217 LD->getChain(),
LD->getBasePtr(),
1218 MemVT,
LD->getMemOperand());
1221 unsigned Opc =
Op.getOpcode();
1225 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1229 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1247 EVT OldVT =
Op.getValueType();
1249 bool Replace =
false;
1250 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1253 AddToWorklist(NewOp.
getNode());
1256 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1262 EVT OldVT =
Op.getValueType();
1264 bool Replace =
false;
1265 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1268 AddToWorklist(NewOp.
getNode());
1271 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1279 if (!LegalOperations)
1282 EVT VT =
Op.getValueType();
1288 unsigned Opc =
Op.getOpcode();
1296 assert(PVT != VT &&
"Don't know what type to promote to!");
1300 bool Replace0 =
false;
1302 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1304 bool Replace1 =
false;
1306 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1318 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1321 CombineTo(
Op.getNode(), RV);
1347 if (!LegalOperations)
1350 EVT VT =
Op.getValueType();
1356 unsigned Opc =
Op.getOpcode();
1364 assert(PVT != VT &&
"Don't know what type to promote to!");
1368 bool Replace =
false;
1372 N0 = SExtPromoteOperand(N0, PVT);
1374 N0 = ZExtPromoteOperand(N0, PVT);
1376 N0 = PromoteOperand(N0, PVT, Replace);
1386 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1396 if (!LegalOperations)
1399 EVT VT =
Op.getValueType();
1405 unsigned Opc =
Op.getOpcode();
1413 assert(PVT != VT &&
"Don't know what type to promote to!");
1423 bool DAGCombiner::PromoteLoad(
SDValue Op) {
1424 if (!LegalOperations)
1430 EVT VT =
Op.getValueType();
1436 unsigned Opc =
Op.getOpcode();
1444 assert(PVT != VT &&
"Don't know what type to promote to!");
1449 EVT MemVT =
LD->getMemoryVT();
1451 :
LD->getExtensionType();
1453 LD->getChain(),
LD->getBasePtr(),
1454 MemVT,
LD->getMemOperand());
1458 Result.getNode()->dump(&DAG);
dbgs() <<
'\n');
1459 WorklistRemover DeadNodes(*
this);
1462 deleteAndRecombine(
N);
1463 AddToWorklist(
Result.getNode());
1475 bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1476 if (!
N->use_empty())
1486 if (
N->use_empty()) {
1487 for (
const SDValue &ChildN :
N->op_values())
1488 Nodes.
insert(ChildN.getNode());
1495 }
while (!Nodes.
empty());
1510 WorklistInserter AddNodes(*
this);
1514 AddToWorklist(&Node);
1522 while (
SDNode *
N = getNextWorklistEntry()) {
1526 if (recursivelyDeleteUnusedNodes(
N))
1529 WorklistRemover DeadNodes(*
this);
1537 for (
SDNode *LN : UpdatedNodes)
1538 AddToWorklistWithUsers(LN);
1550 for (
const SDValue &ChildN :
N->op_values())
1551 if (!CombinedNodes.
count(ChildN.getNode()))
1552 AddToWorklist(ChildN.getNode());
1570 "Node was deleted but visit returned new node!");
1578 N->getNumValues() == 1 &&
"Type mismatch");
1589 AddUsersToWorklist(RV.
getNode());
1596 recursivelyDeleteUnusedNodes(
N);
1605 switch (
N->getOpcode()) {
1745 if (!DisableGenericCombines)
1751 "Node was deleted but visit returned NULL!");
1758 DagCombineInfo(DAG,
Level,
false,
this);
1766 switch (
N->getOpcode()) {
1774 RV = PromoteIntBinOp(
SDValue(
N, 0));
1779 RV = PromoteIntShiftOp(
SDValue(
N, 0));
1796 N->getNumValues() == 1) {
1801 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1816 if (
unsigned NumOps =
N->getNumOperands()) {
1817 if (
N->getOperand(0).getValueType() ==
MVT::Other)
1818 return N->getOperand(0);
1819 if (
N->getOperand(NumOps-1).getValueType() ==
MVT::Other)
1820 return N->getOperand(NumOps-1);
1821 for (
unsigned i = 1; i < NumOps-1; ++i)
1822 if (
N->getOperand(i).getValueType() ==
MVT::Other)
1823 return N->getOperand(i);
1831 if (
N->getNumOperands() == 2) {
1833 return N->getOperand(0);
1835 return N->getOperand(1);
1850 AddToWorklist(*(
N->use_begin()));
1855 bool Changed =
false;
1862 for (
unsigned i = 0; i < TFs.
size(); ++i) {
1867 for (
unsigned j = i; j < TFs.
size(); j++)
1878 switch (
Op.getOpcode()) {
1896 if (SeenOps.
insert(
Op.getNode()).second)
1907 for (
unsigned i = 1,
e = TFs.
size(); i <
e; i++)
1908 AddToWorklist(TFs[i]);
1920 bool DidPruneOps =
false;
1922 unsigned NumLeftToConsider = 0;
1924 Worklist.
push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
1928 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
1934 unsigned OrigOpNumber = 0;
1935 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() !=
Op)
1937 assert((OrigOpNumber != Ops.size()) &&
1938 "expected to find TokenFactor Operand");
1940 for (
unsigned i = CurIdx + 1; i < Worklist.
size(); ++i) {
1941 if (Worklist[i].second == OrigOpNumber) {
1942 Worklist[i].second = OpNumber;
1945 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1946 OpWorkCount[OrigOpNumber] = 0;
1947 NumLeftToConsider--;
1950 if (SeenChains.
insert(
Op).second) {
1951 OpWorkCount[OpNumber]++;
1956 for (
unsigned i = 0; i < Worklist.
size() && i < 1024; ++i) {
1958 if (NumLeftToConsider <= 1)
1960 auto CurNode = Worklist[i].first;
1961 auto CurOpNumber = Worklist[i].second;
1962 assert((OpWorkCount[CurOpNumber] > 0) &&
1963 "Node should not appear in worklist");
1964 switch (CurNode->getOpcode()) {
1970 NumLeftToConsider++;
1973 for (
const SDValue &
Op : CurNode->op_values())
1974 AddToWorklist(i,
Op.getNode(), CurOpNumber);
1980 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1983 if (
auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1984 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1987 OpWorkCount[CurOpNumber]--;
1988 if (OpWorkCount[CurOpNumber] == 0)
1989 NumLeftToConsider--;
2003 if (SeenChains.
count(
Op.getNode()) == 0)
2018 WorklistRemover DeadNodes(*
this);
2024 AddUsersToWorklist(
N);
2028 for (
unsigned i = 0,
e =
N->getNumOperands(); i !=
e; ++i)
2031 }
while (!
N->use_empty());
2032 deleteAndRecombine(
N);
2040 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2051 if (
LD->isIndexed() ||
LD->getBasePtr().getNode() !=
N)
2053 VT =
LD->getMemoryVT();
2054 AS =
LD->getAddressSpace();
2056 if (
ST->isIndexed() ||
ST->getBasePtr().getNode() !=
N)
2058 VT =
ST->getMemoryVT();
2059 AS =
ST->getAddressSpace();
2061 if (
LD->isIndexed() ||
LD->getBasePtr().getNode() !=
N)
2063 VT =
LD->getMemoryVT();
2064 AS =
LD->getAddressSpace();
2066 if (
ST->isIndexed() ||
ST->getBasePtr().getNode() !=
N)
2068 VT =
ST->getMemoryVT();
2069 AS =
ST->getAddressSpace();
2079 AM.BaseOffs =
Offset->getSExtValue();
2083 }
else if (
N->getOpcode() ==
ISD::SUB) {
2084 AM.HasBaseReg =
true;
2088 AM.BaseOffs = -
Offset->getSExtValue();
2101 "Unexpected binary operator");
2106 unsigned SelOpNo = 0;
2132 bool CanFoldNonConst =
2138 if (!CanFoldNonConst &&
2150 : DAG.
getNode(BinOpcode,
DL, VT, CT, CBO);
2151 if (!CanFoldNonConst && !NewCT.
isUndef() &&
2157 : DAG.
getNode(BinOpcode,
DL, VT, CF, CBO);
2158 if (!CanFoldNonConst && !NewCF.
isUndef() &&
2170 "Expecting add or sub");
2175 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2176 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2177 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2178 auto *CN = dyn_cast<ConstantSDNode>(
C);
2183 if (Z.getOperand(0).getOpcode() !=
ISD::SETCC ||
2184 Z.getOperand(0).getValueType() !=
MVT::i1)
2188 SDValue SetCC = Z.getOperand(0);
2199 EVT VT =
C.getValueType();
2211 "Expecting add or sub");
2215 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2216 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2217 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2242 {ConstantOp, DAG.getConstant(1, DL, VT)}))
2258 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
2293 assert(Sub &&
"Constant folding failed");
2302 assert(Add &&
"Constant folding failed");
2313 if ((!LegalOperations ||
2316 X.getScalarValueSizeInBits() == 1) {
2328 {N1, N0.getOperand(1)}))
2333 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2337 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD,
DL, N0, N1)) {
2404 return (!Max && !
Op) ||
2405 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2459 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2462 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2474 if (
SDValue Combined = visitADDLike(
N))
2509 unsigned Opcode =
N->getOpcode();
2533 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
2551 bool Masked =
false;
2683 DAG.
getVTList(VT, Carry.getValueType()), N0,
2696 if (!
N->hasAnyUseOfValue(1))
2729 if (Force && isa<ConstantSDNode>(V))
2741 bool IsFlip =
false;
2744 IsFlip = Const->isOne();
2747 IsFlip = Const->isAllOnesValue();
2750 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
2767 EVT CarryVT =
N->getValueType(1);
2771 if (!
N->hasAnyUseOfValue(1))
2778 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
2798 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
2801 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
2835 SDValue CarryIn =
N->getOperand(2);
2854 SDValue CarryIn =
N->getOperand(2);
2865 if (!LegalOperations ||
2875 AddToWorklist(CarryExt.
getNode());
2881 if (
SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn,
N))
2884 if (
SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn,
N))
2893 SDValue CarryIn =
N->getOperand(2);
2904 if (!LegalOperations ||
3047 unsigned CarryInOperandNum =
3049 if (Opcode ==
ISD::USUBO && CarryInOperandNum != 1)
3147 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
3164 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3191 if (
N->getFlags().hasNoUnsignedWrap())
3197 if (
N->getFlags().hasNoSignedWrap())
3238 assert(NewC &&
"Constant folding failed");
3248 assert(NewC &&
"Constant folding failed");
3259 assert(NewC &&
"Constant folding failed");
3269 assert(NewC &&
"Constant folding failed");
3405 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3422 if (GA->getGlobal() == GB->getGlobal())
3423 return DAG.
getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
3462 DAG.
getVTList(VT, Carry.getValueType()), NegX, Zero,
3511 if (!
N->hasAnyUseOfValue(1))
3538 EVT CarryVT =
N->getValueType(1);
3542 if (!
N->hasAnyUseOfValue(1))
3574 SDValue CarryIn =
N->getOperand(2);
3586 SDValue CarryIn =
N->getOperand(2);
3590 if (!LegalOperations ||
3601 SDValue CarryIn =
N->getOperand(2);
3605 if (!LegalOperations ||
3646 bool N1IsConst =
false;
3647 bool N1IsOpaqueConst =
false;
3652 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
3658 "Splat APInt should be element width");
3660 N1IsConst = isa<ConstantSDNode>(N1);
3662 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
3663 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
3684 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3699 SDValue LogBase2 = BuildLogBase2(N1,
DL);
3706 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
3707 unsigned Log2Val = (-ConstValue1).logBase2();
3742 if ((MulC - 1).isPowerOf2())
3744 else if ((MulC + 1).isPowerOf2())
3749 MathOp ==
ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
3752 "multiply-by-constant generated out of bounds shift");
3757 TZeros ? DAG.
getNode(MathOp,
DL, VT, Shl,
3779 SDValue Sh(
nullptr, 0),
Y(
nullptr, 0);
3802 isMulAddWithConstProfitable(
N, N0, N1))
3813 const APInt &C1 = NC1->getAPIntValue();
3826 if (!V || V->isNullValue()) {
3841 for (
unsigned I = 0;
I != NumElts; ++
I)
3862 switch (
NodeType.getSimpleVT().SimpleTy) {
3863 default:
return false;
3864 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
3865 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
3866 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
3867 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
3868 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
3876 if (Node->use_empty())
3879 unsigned Opcode = Node->getOpcode();
3884 EVT VT = Node->getValueType(0);
3898 unsigned OtherOpcode = 0;
3909 SDValue Op0 = Node->getOperand(0);
3910 SDValue Op1 = Node->getOperand(1);
3921 unsigned UserOpc =
User->getOpcode();
3922 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
3926 if (UserOpc == OtherOpcode) {
3928 combined = DAG.
getNode(DivRemOpc,
SDLoc(Node), VTs, Op0, Op1);
3929 }
else if (UserOpc == DivRemOpc) {
3932 assert(UserOpc == Opcode);
3937 CombineTo(
User, combined);
3948 EVT VT =
N->getValueType(0);
3951 unsigned Opc =
N->getOpcode();
3960 if (DAG.
isUndef(Opc, {N0, N1}))
3994 EVT VT =
N->getValueType(0);
3999 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
4022 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4030 if (
SDValue V = visitSDIVLike(N0, N1,
N)) {
4037 AddToWorklist(
Mul.getNode());
4039 CombineTo(RemNode, Sub);
4057 EVT VT =
N->getValueType(0);
4064 if (
C->isNullValue() ||
C->isOpaque())
4066 if (
C->getAPIntValue().isPowerOf2())
4068 if ((-
C->getAPIntValue()).isPowerOf2())
4094 AddToWorklist(Sign.
getNode());
4100 AddToWorklist(
Add.getNode());
4111 Sra = DAG.
getSelect(
DL, VT, IsOneOrAllOnes, N0, Sra);
4139 EVT VT =
N->getValueType(0);
4144 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
4163 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4166 if (
SDValue V = visitUDIVLike(N0, N1,
N)) {
4173 AddToWorklist(
Mul.getNode());
4175 CombineTo(RemNode, Sub);
4193 EVT VT =
N->getValueType(0);
4198 SDValue LogBase2 = BuildLogBase2(N1,
DL);
4199 AddToWorklist(LogBase2.
getNode());
4203 AddToWorklist(Trunc.
getNode());
4212 SDValue LogBase2 = BuildLogBase2(N10,
DL);
4213 AddToWorklist(LogBase2.
getNode());
4217 AddToWorklist(Trunc.
getNode());
4219 AddToWorklist(
Add.getNode());
4236 unsigned Opcode =
N->getOpcode();
4239 EVT VT =
N->getValueType(0);
4258 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4271 AddToWorklist(
Add.getNode());
4279 AddToWorklist(
Add.getNode());
4295 isSigned ? visitSDIVLike(N0, N1,
N) : visitUDIVLike(N0, N1,
N);
4301 CombineTo(DivNode, OptimizedDiv);
4304 AddToWorklist(OptimizedDiv.
getNode());
4305 AddToWorklist(
Mul.getNode());
4312 return DivRem.getValue(1);
4320 EVT VT =
N->getValueType(0);
4368 EVT VT =
N->getValueType(0);
4393 SDValue LogBase2 = BuildLogBase2(N1,
DL);
4425 SDValue DAGCombiner::SimplifyNodeWithTwoResults(
SDNode *
N,
unsigned LoOp,
4428 bool HiExists =
N->hasAnyUseOfValue(1);
4429 if (!HiExists && (!LegalOperations ||
4432 return CombineTo(
N, Res, Res);
4436 bool LoExists =
N->hasAnyUseOfValue(0);
4437 if (!LoExists && (!LegalOperations ||
4440 return CombineTo(
N, Res, Res);
4444 if (LoExists && HiExists)
4450 AddToWorklist(
Lo.getNode());
4453 (!LegalOperations ||
4455 return CombineTo(
N, LoOpt, LoOpt);
4460 AddToWorklist(
Hi.getNode());
4463 (!LegalOperations ||
4465 return CombineTo(
N, HiOpt, HiOpt);
4475 EVT VT =
N->getValueType(0);
4495 return CombineTo(
N,
Lo,
Hi);
4506 EVT VT =
N->getValueType(0);
4512 return CombineTo(
N, Zero, Zero);
4518 return CombineTo(
N,
N->getOperand(0), Zero);
4538 return CombineTo(
N,
Lo,
Hi);
4551 EVT CarryVT =
N->getValueType(1);
4557 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
4566 if (C2->getAPIntValue() == 2)
4568 N->getVTList(), N0, N0);
4577 unsigned Opcode =
N->getOpcode();
4581 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
4619 SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(
SDNode *
N) {
4620 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
4622 unsigned LogicOpcode =
N->getOpcode();
4625 LogicOpcode ==
ISD::XOR) &&
"Expected logic opcode");
4638 EVT XVT =
X.getValueType();
4647 if (XVT !=
Y.getValueType())
4651 if ((VT.
isVector() || LegalOperations) &&
4661 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
4671 if (XVT !=
Y.getValueType())
4683 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
4704 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
4717 if (XVT.
isInteger() && XVT ==
Y.getValueType() &&
4721 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
4738 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
4739 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
4740 assert(
X.getValueType() ==
Y.getValueType() &&
4741 "Inputs to shuffles are not the same type");
4747 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
4748 !SVN0->getMask().equals(SVN1->getMask()))
4784 SDValue LL, LR, RL, RR, N0CC, N1CC;
4785 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
4786 !isSetCCEquivalent(N1, RL, RR, N1CC))
4790 "Unexpected operand types for bitwise logic op");
4793 "Unexpected operand types for setcc");
4809 if (LR == RR && CC0 == CC1 && IsInteger) {
4814 bool AndEqZero = IsAnd && CC1 ==
ISD::SETEQ && IsZero;
4816 bool AndGtNeg1 = IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
4818 bool OrNeZero = !IsAnd && CC1 ==
ISD::SETNE && IsZero;
4820 bool OrLtZero = !IsAnd && CC1 ==
ISD::SETLT && IsZero;
4826 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
4828 AddToWorklist(
Or.getNode());
4833 bool AndEqNeg1 = IsAnd && CC1 ==
ISD::SETEQ && IsNeg1;
4835 bool AndLtZero = IsAnd && CC1 ==
ISD::SETLT && IsZero;
4837 bool OrNeNeg1 = !IsAnd && CC1 ==
ISD::SETNE && IsNeg1;
4839 bool OrGtNeg1 = !IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
4845 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
4847 AddToWorklist(
And.getNode());
4861 AddToWorklist(
Add.getNode());
4893 if ((C0Val - C1Val).isPowerOf2()) {
4908 if (LL == RR && LR == RL) {
4915 if (LL == RL && LR == RR) {
4919 (!LegalOperations ||
4940 if (
SDValue V = foldLogicOfSetCCs(
true, N0, N1,
DL))
4952 APInt SRLC = SRLI->getAPIntValue();
4953 if (
ADDC.getMinSignedBits() <= 64 &&
4965 CombineTo(N0.
getNode(), NewAdd);
4982 const APInt &AndMask = CAnd->getAPIntValue();
4995 (ShiftBits + MaskBits <=
Size / 2) &&
5028 EVT LoadResultTy,
EVT &ExtVT) {
5037 if (ExtVT == LoadedVT &&
5038 (!LegalOperations ||
5054 if (LegalOperations &&
5064 bool DAGCombiner::isLegalNarrowLdSt(
LSBaseSDNode *LDST,
5090 if (LdStMemVT.
bitsLT(MemVT))
5095 assert(ShAmt % 8 == 0 &&
"ShAmt is byte offset");
5096 const unsigned ByteShAmt = ShAmt / 8;
5110 if (isa<LoadSDNode>(LDST)) {
5117 if (LegalOperations &&
5126 if (
Load->getNumValues() > 2)
5139 assert(isa<StoreSDNode>(LDST) &&
"It is not a Load nor a Store SDNode");
5145 if (LegalOperations &&
5152 bool DAGCombiner::SearchForAndLoads(
SDNode *
N,
5160 if (
Op.getValueType().isVector())
5164 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
5166 (
Mask->getAPIntValue() &
C->getAPIntValue()) !=
C->getAPIntValue())
5171 if (!
Op.hasOneUse())
5174 switch(
Op.getOpcode()) {
5176 auto *
Load = cast<LoadSDNode>(
Op);
5178 if (isAndLoadExtLoad(
Mask,
Load,
Load->getValueType(0), ExtVT) &&
5196 unsigned ActiveBits =
Mask->getAPIntValue().countTrailingOnes();
5199 cast<VTSDNode>(
Op.getOperand(1))->getVT() :
5200 Op.getOperand(0).getValueType();
5211 if (!SearchForAndLoads(
Op.getNode(), Loads, NodesWithConsts,
Mask,
5222 NodeToMask =
Op.getNode();
5225 for (
unsigned i = 0,
e = NodeToMask->
getNumValues(); i <
e; ++i) {
5229 NodeToMask =
nullptr;
5241 bool DAGCombiner::BackwardsPropagateMask(
SDNode *
N) {
5242 auto *
Mask = dyn_cast<ConstantSDNode>(
N->getOperand(1));
5246 if (!
Mask->getAPIntValue().isMask())
5250 if (isa<LoadSDNode>(
N->getOperand(0)))
5255 SDNode *FixupNode =
nullptr;
5256 if (SearchForAndLoads(
N, Loads, NodesWithConsts,
Mask, FixupNode)) {
5257 if (Loads.
size() == 0)
5269 SDValue(FixupNode, 0), MaskOp);
5276 for (
auto *LogicN : NodesWithConsts) {
5280 if (isa<ConstantSDNode>(Op0))
5290 for (
auto *
Load : Loads) {
5298 SDValue NewLoad = ReduceLoadWidth(
And.getNode());
5300 "Shouldn't be masking the load if it can't be narrowed");
5314 SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(
SDNode *
N) {
5325 unsigned OuterShift;
5326 unsigned InnerShift;
5328 auto matchMask = [&OuterShift, &InnerShift, &
Y](
SDValue M) ->
bool {
5331 OuterShift =
M->getOpcode();
5340 Y =
M->getOperand(1);
5347 else if (matchMask(N0))
5353 EVT VT =
N->getValueType(0);
5370 EVT VT =
And->getValueType(0);
5377 SDValue Not =
And->getOperand(0), And1 =
And->getOperand(1);
5400 if (ShiftAmt.
uge(VTBitWidth))
5427 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
5447 auto *MLoad = dyn_cast<MaskedLoadSDNode>(N0);
5448 auto *BVec = dyn_cast<BuildVectorSDNode>(N1);
5449 if (MLoad && BVec && MLoad->getExtensionType() ==
ISD::EXTLOAD &&
5451 EVT LoadVT = MLoad->getMemoryVT();
5458 uint64_t ElementSize =
5460 if (Splat->getAPIntValue().isMask(ElementSize)) {
5462 ExtVT,
SDLoc(
N), MLoad->getChain(), MLoad->getBasePtr(),
5463 MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(),
5464 LoadVT, MLoad->getMemOperand(), MLoad->getAddressingMode(),
5492 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
5501 if (
SDValue Shuffle = XformToShuffleWithZero(
N))
5509 return RHS->getAPIntValue().isSubsetOf(LHS->
getAPIntValue());
5529 CombineTo(N0.
getNode(), Zext);
5555 APInt SplatValue, SplatUndef;
5556 unsigned SplatBitSize;
5558 bool IsSplat =
Vector->isConstantSplat(SplatValue, SplatUndef,
5559 SplatBitSize, HasAnyUndefs);
5563 SplatValue |= SplatUndef;
5569 unsigned EltBitWidth =
Vector->getValueType(0).getScalarSizeInBits();
5574 if (EltBitWidth > SplatBitSize)
5575 for (SplatValue = SplatValue.
zextOrTrunc(EltBitWidth);
5576 SplatBitSize < EltBitWidth; SplatBitSize = SplatBitSize * 2)
5577 SplatValue |= SplatValue.
shl(SplatBitSize);
5581 if ((SplatBitSize % EltBitWidth) == 0) {
5583 for (
unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
5593 Load->getValueType(0),
5594 Load->getMemoryVT());
5602 switch (
Load->getExtensionType()) {
5603 default:
B =
false;
break;
5620 Load->getChain(),
Load->getBasePtr(),
5621 Load->getOffset(),
Load->getMemoryVT(),
5622 Load->getMemOperand());
5624 if (
Load->getNumValues() == 3) {
5628 CombineTo(
Load, To, 3,
true);
5630 CombineTo(
Load, NewLoad.getValue(0), NewLoad.getValue(1));
5639 if (
auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
5640 EVT MemVT = GN0->getMemoryVT();
5646 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
5647 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
5653 CombineTo(
N, ZExtLoad);
5654 AddToWorklist(ZExtLoad.
getNode());
5666 if (
SDValue Res = ReduceLoadWidth(
N)) {
5680 if (BackwardsPropagateMask(
N))
5684 if (
SDValue Combined = visitANDLike(N0, N1,
N))
5689 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
5729 ((!LegalOperations && LN0->
isSimple()) ||
5747 if (
SDValue Shifts = unfoldExtremeBitClearingToShifts(
N))
5764 auto *
C = dyn_cast<ConstantSDNode>(RHS);
5768 if (!
C->getAPIntValue().isMask(
5776 if (IsAndZeroExtMask(N0, N1))
5784 bool DemandHighBits) {
5785 if (!LegalOperations)
5788 EVT VT =
N->getValueType(0);
5795 bool LookPassAnd0 =
false;
5796 bool LookPassAnd1 =
false;
5811 LookPassAnd0 =
true;
5821 LookPassAnd1 =
true;
5847 LookPassAnd0 =
true;
5861 LookPassAnd1 =
true;
5870 if (DemandHighBits && OpSizeInBits > 16) {
5879 if (!LookPassAnd1 &&
5886 if (OpSizeInBits > 16) {
5902 if (!
N.getNode()->hasOneUse())
5905 unsigned Opc =
N.getOpcode();
5917 N1C = dyn_cast<ConstantSDNode>(
N.getOperand(1));
5919 N1C = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
5923 unsigned MaskByteOffset;
5927 case 0xFF: MaskByteOffset = 0;
break;
5928 case 0xFF00: MaskByteOffset = 1;
break;
5937 case 0xFF0000: MaskByteOffset = 2;
break;
5938 case 0xFF000000: MaskByteOffset = 3;
break;
5943 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
5949 if (!
C ||
C->getZExtValue() != 8)
5957 if (!
C ||
C->getZExtValue() != 8)
5963 if (MaskByteOffset != 0 && MaskByteOffset != 2)
5966 if (!
C ||
C->getZExtValue() != 8)
5971 if (MaskByteOffset != 1 && MaskByteOffset != 3)
5974 if (!
C ||
C->getZExtValue() != 8)
5978 if (Parts[MaskByteOffset])
5993 if (!
C ||
C->getAPIntValue() != 16)
5995 Parts[0] = Parts[1] =
N.getOperand(0).getOperand(0).getNode();
6010 "MatchBSwapHWordOrAndAnd: expecting i32");
6020 if (!Mask0 || !Mask1)
6031 if (!ShiftAmt0 || !ShiftAmt1)
6051 if (!LegalOperations)
6054 EVT VT =
N->getValueType(0);
6093 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
6122 if (
SDValue V = foldLogicOfSetCCs(
false, N0, N1,
DL))
6137 const APInt &LHSMask = N0O1C->getAPIntValue();
6138 const APInt &RHSMask = N1O1C->getAPIntValue();
6193 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
6212 if (isa<ShuffleVectorSDNode>(N0) &&
6213 isa<ShuffleVectorSDNode>(N1) &&
6221 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
6222 assert((!ZeroN00 || !ZeroN01) &&
"Both inputs zero!");
6223 assert((!ZeroN10 || !ZeroN11) &&
"Both inputs zero!");
6226 bool CanFold =
true;
6230 for (
int i = 0; i != NumElts; ++i) {
6235 bool M0Zero =
M0 < 0 || (ZeroN00 == (
M0 < NumElts));
6236 bool M1Zero =
M1 < 0 || (ZeroN10 == (
M1 < NumElts));
6240 if ((M0Zero &&
M1 < 0) || (M1Zero &&
M0 < 0)) {
6246 if (M0Zero == M1Zero) {
6251 assert((
M0 >= 0 ||
M1 >= 0) &&
"Undef index!");
6257 Mask[i] = M1Zero ?
M0 % NumElts : (
M1 % NumElts) + NumElts;
6268 return LegalShuffle;
6292 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
6299 if (
SDValue Combined = visitORLike(N0, N1,
N))
6306 if (
SDValue BSwap = MatchBSwapHWord(
N, N0, N1))
6308 if (
SDValue BSwap = MatchBSwapHWordLow(
N, N0, N1))
6337 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
6354 if (
SDValue Combined = visitADDLike(
N))
6364 return Op.getOperand(0);
6407 assert(OppShift && ExtractFrom &&
"Empty SDValue");
6410 "Existing shift must be valid as a rotate half");
6436 bool IsMulOrDiv =
false;
6439 auto SelectOpcode = [&](
unsigned NeededShift,
unsigned MulOrDivVariant) {
6440 IsMulOrDiv = ExtractFrom.
getOpcode() == MulOrDivVariant;
6441 if (!IsMulOrDiv && ExtractFrom.
getOpcode() != NeededShift)
6443 Opcode = NeededShift;
6493 if (Rem != 0 || ResultAmt != OppLHSAmt)
6499 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.
zextOrTrunc(
6508 return DAG.
getNode(Opcode,
DL, ResVT, OppShiftLHS, NewShiftNode);
6559 unsigned MaskLoBits = 0;
6564 if (NegC->getAPIntValue().getActiveBits() <=
Bits &&
6565 ((NegC->getAPIntValue() | Known.
Zero).countTrailingOnes() >=
Bits)) {
6585 if (PosC->getAPIntValue().getActiveBits() <= MaskLoBits &&
6586 ((PosC->getAPIntValue() | Known.
Zero).countTrailingOnes() >=
6605 if ((Pos == NegOp1) ||
6629 return Width.getLoBits(MaskLoBits) == 0;
6630 return Width == EltSize;
6640 SDValue InnerNeg,
unsigned PosOpcode,
6641 unsigned NegOpcode,
const SDLoc &
DL) {
6653 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, Shifted,
6654 HasPos ? Pos : Neg);
6668 SDValue InnerNeg,
unsigned PosOpcode,
6669 unsigned NegOpcode,
const SDLoc &
DL) {
6680 if (
matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, N0 == N1)) {
6682 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, N0, N1,
6683 HasPos ? Pos : Neg);
6690 auto IsBinOpImm = [](
SDValue Op,
unsigned BinOpc,
unsigned Imm) {
6691 if (
Op.getOpcode() != BinOpc)
6700 IsBinOpImm(InnerNeg,
ISD::XOR, EltBits - 1) &&
6709 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
6719 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
6740 bool HasROTL = hasOperation(
ISD::ROTL, VT);
6741 bool HasROTR = hasOperation(
ISD::ROTR, VT);
6742 bool HasFSHL = hasOperation(
ISD::FSHL, VT);
6743 bool HasFSHR = hasOperation(
ISD::FSHR, VT);
6744 if (!HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
6766 if (!LHSShift && !RHSShift)
6781 RHSShift = NewRHSShift;
6786 LHSShift = NewLHSShift;
6789 if (!RHSShift || !LHSShift)
6798 if (!IsRotate && !(HasFSHL || HasFSHR))
6821 return (LHS->
getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
6825 if (IsRotate && (HasROTL || HasROTR))
6827 HasROTL ? LHSShiftAmt : RHSShiftAmt);
6830 RHSShiftArg, HasFSHL ? LHSShiftAmt : RHSShiftAmt);
6860 SDValue LExtOp0 = LHSShiftAmt;
6861 SDValue RExtOp0 = RHSShiftAmt;
6874 if (IsRotate && (HasROTL || HasROTR)) {
6876 MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt, LExtOp0,
6882 MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt, RExtOp0,
6889 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt, RHSShiftAmt,
6895 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
6907 struct ByteProvider {
6912 unsigned ByteOffset = 0;
6914 ByteProvider() =
default;
6917 return ByteProvider(
Load, ByteOffset);
6920 static ByteProvider getConstantZero() {
return ByteProvider(
nullptr, 0); }
6922 bool isConstantZero()
const {
return !
Load; }
6923 bool isMemory()
const {
return Load; }
6925 bool operator==(
const ByteProvider &Other)
const {
6931 :
Load(
Load), ByteOffset(ByteOffset) {}
6949 bool Root =
false) {
6954 if (!Root && !
Op.hasOneUse())
6957 assert(
Op.getValueType().isScalarInteger() &&
"can't handle other types");
6962 assert(
Index < ByteWidth &&
"invalid index requested");
6965 switch (
Op.getOpcode()) {
6974 if (LHS->isConstantZero())
6976 if (RHS->isConstantZero())
6981 auto ShiftOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
6985 uint64_t BitShift = ShiftOp->getZExtValue();
6986 if (BitShift % 8 != 0)
6988 uint64_t ByteShift = BitShift / 8;
6990 return Index < ByteShift
6991 ? ByteProvider::getConstantZero()
7000 if (NarrowBitWidth % 8 != 0)
7002 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
7004 if (
Index >= NarrowByteWidth)
7014 auto L = cast<LoadSDNode>(
Op.getNode());
7015 if (!L->isSimple() || L->isIndexed())
7018 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
7019 if (NarrowBitWidth % 8 != 0)
7021 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
7023 if (
Index >= NarrowByteWidth)
7046 int64_t FirstOffset) {
7052 bool BigEndian =
true, LittleEndian =
true;
7053 for (
unsigned i = 0; i <
Width; i++) {
7054 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
7057 if (!BigEndian && !LittleEndian)
7061 assert((BigEndian != LittleEndian) &&
"It should be either big endian or"
7067 switch (
Value.getOpcode()) {
7105 if (LegalOperations)
7110 EVT MemVT =
N->getMemoryVT();
7112 !
N->isSimple() ||
N->isIndexed())
7118 while (
auto *
Store = dyn_cast<StoreSDNode>(Chain)) {
7122 if (
Store->getMemoryVT() != MemVT || !
Store->isSimple() ||
7126 Chain =
Store->getChain();
7129 if (Stores.
size() < 2)
7134 unsigned NumStores = Stores.
size();
7135 unsigned NarrowNumBits =
N->getMemoryVT().getScalarSizeInBits();
7136 unsigned WideNumBits = NumStores * NarrowNumBits;
7148 for (
auto Store : Stores) {
7159 isa<ConstantSDNode>(WideVal.
getOperand(1))) {
7167 if (ShiftAmtC % NarrowNumBits != 0)
7170 Offset = ShiftAmtC / NarrowNumBits;
7177 SourceValue = WideVal;
7184 SourceValue = WideVal;
7192 int64_t ByteOffsetFromBase = 0;
7195 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
7199 if (ByteOffsetFromBase < FirstOffset) {
7201 FirstOffset = ByteOffsetFromBase;
7205 if (Offset < 0 || Offset >= NumStores || OffsetMap[
Offset] !=
INT64_MAX)
7207 OffsetMap[
Offset] = ByteOffsetFromBase;
7211 assert(FirstStore &&
"First store must be set");
7223 auto checkOffsets = [&](
bool MatchLittleEndian) {
7224 if (MatchLittleEndian) {
7225 for (
unsigned i = 0; i != NumStores; ++i)
7226 if (OffsetMap[i] != i * (NarrowNumBits / 8) + FirstOffset)
7229 for (
unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --j)
7230 if (OffsetMap[j] != i * (NarrowNumBits / 8) + FirstOffset)
7237 bool NeedBswap =
false;
7238 bool NeedRotate =
false;
7241 if (NarrowNumBits == 8 && checkOffsets(Layout.
isBigEndian()))
7243 else if (NumStores == 2 && checkOffsets(Layout.
isBigEndian()))
7252 "Unexpected store value to merge");
7261 }
else if (NeedRotate) {
7262 assert(WideNumBits % 2 == 0 &&
"Unexpected type for rotate");
7308 "Can only match load combining against OR nodes");
7311 EVT VT =
N->getValueType(0);
7317 auto MemoryByteOffset = [&] (ByteProvider
P) {
7318 assert(
P.isMemory() &&
"Must be a memory byte provider");
7319 unsigned LoadBitWidth =
P.Load->getMemoryVT().getSizeInBits();
7320 assert(LoadBitWidth % 8 == 0 &&
7321 "can only analyze providers for individual bytes not bit");
7322 unsigned LoadByteWidth = LoadBitWidth / 8;
7323 return IsBigEndianTarget
7338 unsigned ZeroExtendedBytes = 0;
7339 for (
int i = ByteWidth - 1; i >= 0; --i) {
7344 if (
P->isConstantZero()) {
7347 if (++ZeroExtendedBytes != (ByteWidth -
static_cast<unsigned>(i)))
7351 assert(
P->isMemory() &&
"provenance should either be memory or zero");
7356 "Must be enforced by calculateByteProvider");
7363 else if (Chain != LChain)
7368 int64_t ByteOffsetFromBase = 0;
7371 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
7375 ByteOffsetFromBase += MemoryByteOffset(*
P);
7376 ByteOffsets[i] = ByteOffsetFromBase;
7379 if (ByteOffsetFromBase < FirstOffset) {
7380 FirstByteProvider =
P;
7381 FirstOffset = ByteOffsetFromBase;
7386 assert(!Loads.
empty() &&
"All the bytes of the value must be loaded from "
7387 "memory, so there must be at least one load which produces the value");
7388 assert(Base &&
"Base address of the accessed memory location must be set");
7391 bool NeedsZext = ZeroExtendedBytes > 0;
7402 if (LegalOperations &&
7410 makeArrayRef(ByteOffsets).drop_back(ZeroExtendedBytes), FirstOffset);
7414 assert(FirstByteProvider &&
"must be set");
7418 if (MemoryByteOffset(*FirstByteProvider) != 0)
7420 LoadSDNode *FirstLoad = FirstByteProvider->Load;
7427 bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
7434 if (NeedsBswap && (LegalOperations || NeedsZext) &&
7440 if (NeedsBswap && NeedsZext && LegalOperations &&
7468 SDLoc(
N), LegalOperations))
7493 EVT VT =
N->getValueType(0);
7515 M =
And.getOperand(XorIdx ? 0 : 1);
7521 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
7522 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
7528 if (isa<ConstantSDNode>(
M.getNode()))
7562 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
7596 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
7607 isSetCCEquivalent(N0, LHS, RHS, CC,
true)) {
7610 if (!LegalOperations ||
7628 CombineTo(
N, SetCC);
7630 recursivelyDeleteUnusedNodes(N0.
getNode());
7641 isSetCCEquivalent(N0.
getOperand(0), LHS, RHS, CC)){
7654 if (isOneUseSetCC(N01) || isOneUseSetCC(N00)) {
7659 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
7666 if (isa<ConstantSDNode>(N01) || isa<ConstantSDNode>(N00)) {
7671 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
7695 AddToWorklist(NotX.
getNode());
7703 if (XorC && ShiftC) {
7709 Ones = N0Opcode ==
ISD::SHL ? Ones.
shl(ShiftAmt) : Ones.
lshr(ShiftAmt);
7726 SDValue A0 =
A.getOperand(0), A1 =
A.getOperand(1);
7728 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
7765 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
7769 if (
SDValue MM = unfoldMaskedMerge(
N))
7792 unsigned LogicOpcode = LogicOp.
getOpcode();
7798 unsigned ShiftOpcode = Shift->
getOpcode();
7801 assert(C1Node &&
"Expected a shift with constant operand");
7804 const APInt *&ShiftAmtVal) {
7818 if (ShiftAmtVal->getBitWidth() != C1Val.
getBitWidth())
7831 if (matchFirstShift(LogicOp.
getOperand(0),
X, C0Val))
7833 else if (matchFirstShift(LogicOp.
getOperand(1),
X, C0Val))
7845 return DAG.
getNode(LogicOpcode,
DL, VT, NewShift1, NewShift2);
7903 isa<ConstantSDNode>(BinOpLHSVal.
getOperand(1));
7907 if (!IsShiftByConstant && !IsCopyOrSelect)
7910 if (IsCopyOrSelect &&
N->hasOneUse())
7915 EVT VT =
N->getValueType(0);
7918 assert(isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!");
7930 EVT TruncVT =
N->getValueType(0);
7931 if (
N->hasOneUse() &&
N->getOperand(0).hasOneUse() &&
7933 SDValue N01 =
N->getOperand(0).getOperand(1);
7936 SDValue N00 =
N->getOperand(0).getOperand(0);
7939 AddToWorklist(Trunc00.
getNode());
7940 AddToWorklist(Trunc01.
getNode());
7952 EVT VT =
N->getValueType(0);
7967 bool OutOfRange =
false;
7969 OutOfRange |=
C->getAPIntValue().uge(Bitsize);
7977 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, Amt);
7982 if (RotAmtC && RotAmtC->getAPIntValue() == 8 &&
7994 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, NewOp1);
8004 bool SameSide = (
N->getOpcode() == NextOp);
8007 CombineOp, dl, ShiftVT, {N1, N0.getOperand(1)})) {
8010 ISD::SREM, dl, ShiftVT, {CombinedShift, BitsizeC});
8031 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
8060 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
8083 APInt c2 = RHS->getAPIntValue();
8085 return (c1 + c2).uge(OpSizeInBits);
8093 APInt c2 = RHS->getAPIntValue();
8095 return (c1 + c2).ult(OpSizeInBits);
8118 auto MatchOutOfRange = [OpSizeInBits, InnerBitwidth](
ConstantSDNode *LHS,
8120 APInt c1 = LHS->getAPIntValue();
8121 APInt c2 = RHS->getAPIntValue();
8123 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
8124 (c1 + c2).uge(OpSizeInBits);
8131 auto MatchInRange = [OpSizeInBits, InnerBitwidth](
ConstantSDNode *LHS,
8133 APInt c1 = LHS->getAPIntValue();
8134 APInt c2 = RHS->getAPIntValue();
8136 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
8137 (c1 + c2).ult(OpSizeInBits);
8160 APInt c2 = RHS->getAPIntValue();
8171 AddToWorklist(NewSHL.
getNode());
8182 uint64_t C1 = N0C1->getZExtValue();
8202 if (N0C1->getAPIntValue().ult(OpSizeInBits)) {
8213 Mask.lshrInPlace(c1 - c2);
8245 AddToWorklist(Shl0.
getNode());
8246 AddToWorklist(Shl1.
getNode());
8260 if (
SDValue NewSHL = visitShiftByConstant(
N))
8267 const APInt &C1 = NC1->getAPIntValue();
8281 "SRL or SRA node is required here!");
8292 SDValue ShiftOperand =
N->getOperand(0);
8309 assert((WideVT1 == WideVT2) &&
8310 "Cannot have a multiply node with two different operand types.");
8327 if (ShiftAmt != NarrowVTSize)
8361 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
8370 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
8381 if (!LegalOperations ||
8398 APInt c2 = RHS->getAPIntValue();
8400 APInt Sum = c1 + c2;
8411 ShiftValue = ShiftValues[0];
8439 if ((ShiftAmt > 0) &&
8451 N->getValueType(0), Trunc);
8508 if (LargeShift->getAPIntValue() == TruncBits) {
8528 if (
SDValue NewSRA = visitShiftByConstant(
N))
8550 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
8559 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
8572 APInt c2 = RHS->getAPIntValue();
8574 return (c1 + c2).uge(OpSizeInBits);
8582 APInt c2 = RHS->getAPIntValue();
8584 return (c1 + c2).ult(OpSizeInBits);
8606 if (c1 + OpSizeInBits == InnerShiftSize) {
8608 if (c1 + c2 >= InnerShiftSize)
8618 c1 + c2 < InnerShiftSize) {
8639 AddToWorklist(
Mask.getNode());
8659 AddToWorklist(SmallShift.
getNode());
8703 AddToWorklist(
Op.getNode());
8725 if (
SDValue NewSRL = visitShiftByConstant(
N))
8729 if (
SDValue NarrowLoad = ReduceLoadWidth(
N))
8749 if (
N->hasOneUse()) {
8770 EVT VT =
N->getValueType(0);
8782 return IsFSHL ? N0 : N1;
8784 auto IsUndefOrZero = [](
SDValue V) {
8801 return IsFSHL ? N0 : N1;
8807 if (IsUndefOrZero(N0))
8811 if (IsUndefOrZero(N1))
8823 auto *LHS = dyn_cast<LoadSDNode>(N0);
8824 auto *RHS = dyn_cast<LoadSDNode>(N1);
8825 if (LHS && RHS && LHS->isSimple() && RHS->isSimple() &&
8826 LHS->getAddressSpace() == RHS->getAddressSpace() &&
8836 RHS->getAddressSpace(), NewAlign,
8837 RHS->getMemOperand()->
getFlags(), &Fast) &&
8841 AddToWorklist(NewPtr.
getNode());
8843 VT,
DL, RHS->getChain(), NewPtr,
8844 RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
8845 RHS->getMemOperand()->
getFlags(), RHS->getAAInfo());
8847 WorklistRemover DeadNodes(*
this);
8873 if (N0 == N1 && hasOperation(RotOpc, VT))
8885 EVT VT =
N->getValueType(0);
8901 EVT VT =
N->getValueType(0);
8914 EVT VT =
N->getValueType(0);
8927 EVT VT =
N->getValueType(0);
8944 EVT VT =
N->getValueType(0);
8954 EVT VT =
N->getValueType(0);
8971 EVT VT =
N->getValueType(0);
8981 EVT VT =
N->getValueType(0);
9007 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
9023 return DAG.
getNode(IEEEOpcode,
DL, VT, LHS, RHS);
9027 return DAG.
getNode(Opcode,
DL, VT, LHS, RHS);
9038 return DAG.
getNode(IEEEOpcode,
DL, VT, LHS, RHS);
9042 return DAG.
getNode(Opcode,
DL, VT, LHS, RHS);
9057 "Expected select-of-constants");
9059 EVT VT =
N->getValueType(0);
9061 VT !=
Cond.getOperand(0).getValueType())
9091 EVT VT =
N->getValueType(0);
9092 EVT CondVT =
Cond.getValueType();
9098 auto *C1 = dyn_cast<ConstantSDNode>(N1);
9099 auto *C2 = dyn_cast<ConstantSDNode>(N2);
9107 if (CondVT ==
MVT::i1 && !LegalOperations) {
9140 const APInt &C1Val = C1->getAPIntValue();
9141 const APInt &C2Val = C2->getAPIntValue();
9142 if (C1Val - 1 == C2Val) {
9148 if (C1Val + 1 == C2Val) {
9199 EVT VT =
N->getValueType(0);
9212 if (
SDValue V = foldSelectOfConstants(
N))
9218 AddToWorklist(NOTNode.
getNode());
9224 AddToWorklist(NOTNode.
getNode());
9233 if (SimplifySelectOps(
N, N1, N2))
9245 bool normalizeToSequence =
9254 if (normalizeToSequence || !InnerSelect.
use_empty())
9256 InnerSelect, N2, Flags);
9259 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
9266 Cond1, N1, N2, Flags);
9267 if (normalizeToSequence || !InnerSelect.
use_empty())
9269 InnerSelect, Flags);
9272 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
9282 if (!normalizeToSequence) {
9288 if (
SDValue Combined = visitANDLike(N0, N1_0,
N)) {
9301 if (!normalizeToSequence) {
9307 if (
SDValue Combined = visitORLike(N0, N2_0,
N))
9343 auto *
C = dyn_cast<ConstantSDNode>(N2.
getOperand(1));
9344 auto *NotC = dyn_cast<ConstantSDNode>(Cond1);
9345 if (
C && NotC &&
C->getAPIntValue() == ~NotC->getAPIntValue()) {
9365 (!LegalOperations &&
9376 return SimplifySelect(
DL, N0, N1, N2);
9389 EVT VT =
N->getValueType(0);
9406 for (
int i = 0; i < NumElems / 2; ++i) {
9407 if (
Cond->getOperand(i)->isUndef())
9410 if (BottomHalf ==
nullptr)
9411 BottomHalf = cast<ConstantSDNode>(
Cond.getOperand(i));
9412 else if (
Cond->getOperand(i).getNode() != BottomHalf)
9418 for (
int i = NumElems / 2; i < NumElems; ++i) {
9419 if (
Cond->getOperand(i)->isUndef())
9422 if (TopHalf ==
nullptr)
9423 TopHalf = cast<ConstantSDNode>(
Cond.getOperand(i));
9424 else if (
Cond->getOperand(i).getNode() != TopHalf)
9428 assert(TopHalf && BottomHalf &&
9429 "One half of the selector was all UNDEFs and the other was all the "
9430 "same value. This should have been addressed before this function.");
9528 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
9546 return CombineTo(
N, PassThru, MGT->
getChain());
9583 return CombineTo(
N, NewLd, NewLd.
getValue(1));
9587 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
9599 EVT VT =
N->getValueType(0);
9600 if (!
Cond.hasOneUse() ||
Cond.getScalarValueSizeInBits() != 1 ||
9609 bool AllAddOne =
true;
9610 bool AllSubOne =
true;
9612 for (
unsigned i = 0; i != Elts; ++i) {
9620 const APInt &C1 = cast<ConstantSDNode>(N1Elt)->getAPIntValue();
9621 const APInt &C2 = cast<ConstantSDNode>(N2Elt)->getAPIntValue();
9631 if (AllAddOne || AllSubOne) {
9662 EVT VT =
N->getValueType(0);
9699 AddToWorklist(Shift.
getNode());
9700 AddToWorklist(
Add.getNode());
9730 SetCCWidth != 1 && SetCCWidth < WideWidth &&
9760 SDValue CondLHS = LHS, CondRHS = RHS;
9773 (OpLHS == CondLHS || OpRHS == CondLHS))
9776 if (isa<BuildVectorSDNode>(OpRHS) && isa<BuildVectorSDNode>(CondRHS) &&
9782 return Cond->getAPIntValue() == ~
Op->getAPIntValue();
9804 if (Other &&
Other.getNumOperands() == 2 &&
Other.getOperand(0) == LHS) {
9815 if (
auto *OpRHSBV = dyn_cast<BuildVectorSDNode>(OpRHS)) {
9816 if (isa<BuildVectorSDNode>(CondRHS)) {
9823 Cond->getAPIntValue() == (-
Op->getAPIntValue() - 1));
9838 if (
auto *OpRHSConst = OpRHSBV->getConstantSplatNode()) {
9841 OpRHSConst->getAPIntValue().isSignMask()) {
9844 OpRHS = DAG.
getConstant(OpRHSConst->getAPIntValue(),
DL, VT);
9854 if (SimplifySelectOps(
N, N1, N2))
9874 if (
SDValue V = foldVSelectOfConstants(
N))
9895 AddToWorklist(SCC.getNode());
9897 if (
ConstantSDNode *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode())) {
9898 if (!SCCC->isNullValue())
9902 }
else if (SCC->isUndef()) {
9910 SCC.getOperand(1), N2, N3, SCC.getOperand(2));
9911 SelectOp->
setFlags(SCC->getFlags());
9917 if (SimplifySelectOps(
N, N2, N3))
9921 return SimplifySelectCC(
SDLoc(
N), N0, N1, N2, N3, CC);
9929 N->hasOneUse() &&
N->use_begin()->getOpcode() ==
ISD::BRCOND;
9932 N->getValueType(0),
N->getOperand(0),
N->getOperand(1),
9933 cast<CondCodeSDNode>(
N->getOperand(2))->get(),
SDLoc(
N), !PreferSetCC);
9941 SDValue NewSetCC = rebuildSetCC(Combined);
9975 unsigned Opcode =
N->getOpcode();
9977 EVT VT =
N->getValueType(0);
9983 &&
"Expected EXTEND dag node in input!");
9988 if (isa<ConstantSDNode>(N0))
9997 if (isa<ConstantSDNode>(Op1) && isa<ConstantSDNode>(Op2) &&
10008 unsigned FoldOpc = Opcode;
10036 for (
unsigned i = 0; i != NumElts; ++i) {
10038 if (
Op.isUndef()) {
10046 APInt C = cast<ConstantSDNode>(
Op)->getAPIntValue().zextOrTrunc(EVTBits);
10064 bool HasCopyToRegUses =
false;
10072 if (UI.getUse().getResNo() != N0.
getResNo())
10081 for (
unsigned i = 0; i != 2; ++i) {
10085 if (!isa<ConstantSDNode>(UseOp))
10099 HasCopyToRegUses =
true;
10102 if (HasCopyToRegUses) {
10103 bool BothLiveOut =
false;
10108 BothLiveOut =
true;
10115 return ExtendNodes.
size();
10125 for (
SDNode *SetCC : SetCCs) {
10128 for (
unsigned j = 0; j != 2; ++j) {
10130 if (SOp == OrigLoad)
10144 EVT DstVT =
N->getValueType(0);
10149 "Unexpected node type (not an extend)!");
10187 EVT SplitSrcVT = SrcVT;
10188 EVT SplitDstVT = DstVT;
10201 const unsigned NumSplits =
10208 for (
unsigned Idx = 0; Idx < NumSplits; Idx++) {
10209 const unsigned Offset = Idx * Stride;
10227 AddToWorklist(NewChain.
getNode());
10229 CombineTo(
N, NewValue);
10235 ExtendSetCCUses(SetCCs, N0, NewValue, (
ISD::NodeType)
N->getOpcode());
10236 CombineTo(N0.
getNode(), Trunc, NewChain);
10244 EVT VT =
N->getValueType(0);
10245 EVT OrigVT =
N->getOperand(0).getValueType();
10268 EVT MemVT =
Load->getMemoryVT();
10289 Load->getChain(),
Load->getBasePtr(),
10290 Load->getMemoryVT(),
Load->getMemOperand());
10307 Load->getValueType(0), ExtLoad);
10312 recursivelyDeleteUnusedNodes(N0.
getNode());
10321 SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(
SDNode *Cast) {
10322 unsigned CastOpcode = Cast->
getOpcode();
10326 "Unexpected opcode for vector select narrowing/widening");
10356 CastA = DAG.
getNode(CastOpcode,
DL, VT, A);
10366 bool LegalOperations,
SDNode *
N,
10377 if ((LegalOperations || !LN0->
isSimple() ||
10388 Combiner.recursivelyDeleteUnusedNodes(LN0);
10402 ((LegalOperations || VT.
isVector() ||
10403 !cast<LoadSDNode>(N0)->isSimple()) &&
10407 bool DoXform =
true;
10420 Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
10424 if (NoReplaceTrunc) {
10426 Combiner.recursivelyDeleteUnusedNodes(LN0);
10464 bool LegalOperations) {
10476 EVT VT =
N->getValueType(0);
10477 EVT XVT =
X.getValueType();
10493 return DAG.
getNode(ShiftOpcode,
DL, VT, NotX, ShiftAmount);
10501 EVT VT =
N->getValueType(0);
10517 if (NarrowLoad.getNode() != N0.
getNode()) {
10518 CombineTo(N0.
getNode(), NarrowLoad);
10520 AddToWorklist(oye);
10528 unsigned OpBits =
Op.getScalarValueSizeInBits();
10533 if (OpBits == DestBits) {
10536 if (NumSignBits > DestBits-MidBits)
10538 }
else if (OpBits < DestBits) {
10541 if (NumSignBits > OpBits-MidBits)
10546 if (NumSignBits > OpBits-MidBits)
10553 if (OpBits < DestBits)
10555 else if (OpBits > DestBits)
10575 if (
SDValue ExtLoad = CombineExtLoad(
N))
10606 bool NoReplaceTruncAnd = !N0.
hasOneUse();
10610 if (NoReplaceTruncAnd) {
10613 CombineTo(N0.
getNode(), TruncAnd);
10615 if (NoReplaceTrunc) {
10620 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
10638 if (VT.
isVector() && !LegalOperations &&
10660 if (SVT == MatchingVecType) {
10679 SDValue ExtTrueVal = (SetCCWidth == 1)
10684 SimplifySelectCC(
DL, N00, N01, ExtTrueVal, Zero, CC,
true))
10696 return DAG.
getSelect(
DL, VT, SetCC, ExtTrueVal, Zero);
10706 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
10737 if (NewXor.getNode() == N0.
getNode()) {
10761 Op =
N->getOperand(0);
10767 N.getValueType().getScalarType() !=
MVT::i1 ||
10768 cast<CondCodeSDNode>(
N.getOperand(2))->get() !=
ISD::SETNE)
10784 return (Known.
Zero | 1).isAllOnesValue();
10812 EVT VT =
N->getValueType(0);
10829 APInt TruncatedBits =
10831 APInt(
Op.getScalarValueSizeInBits(), 0) :
10846 if (NarrowLoad.getNode() != N0.
getNode()) {
10847 CombineTo(N0.
getNode(), NarrowLoad);
10849 AddToWorklist(oye);
10864 AddToWorklist(
Op.getNode());
10868 return ZExtOrTrunc;
10874 AddToWorklist(
Op.getNode());
10912 if (
SDValue ExtLoad = CombineExtLoad(
N))
10928 bool DoXform =
true;
10932 auto *AndC = cast<ConstantSDNode>(N0.
getOperand(1));
10935 if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
10952 bool NoReplaceTruncAnd = !N0.
hasOneUse();
10956 if (NoReplaceTruncAnd) {
10959 CombineTo(N0.
getNode(), TruncAnd);
10961 if (NoReplaceTrunc) {
10966 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
10975 if (
SDValue ZExtLoad = CombineZExtLogicopShiftLoad(
N))
10988 if (!LegalOperations && VT.
isVector() &&
11022 if (
SDValue SCC = SimplifySelectCC(
11026 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
11042 if (cast<ConstantSDNode>(ShAmt)->getAPIntValue().ugt(KnownZeroBits))
11057 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
11068 EVT VT =
N->getValueType(0);
11086 if (NarrowLoad.getNode() != N0.
getNode()) {
11087 CombineTo(N0.
getNode(), NarrowLoad);
11089 AddToWorklist(oye);
11126 bool DoXform =
true;
11139 CombineTo(
N, ExtLoad);
11140 if (NoReplaceTrunc) {
11142 recursivelyDeleteUnusedNodes(LN0);
11146 CombineTo(LN0, Trunc, ExtLoad.
getValue(1));
11160 if (!LegalOperations || TLI.
isLoadExtLegal(ExtType, VT, MemVT)) {
11164 CombineTo(
N, ExtLoad);
11166 recursivelyDeleteUnusedNodes(LN0);
11177 if (VT.
isVector() && !LegalOperations) {
11190 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
11199 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
11205 if (
SDValue SCC = SimplifySelectCC(
11208 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
11219 unsigned Opcode =
N->getOpcode();
11222 EVT AssertVT = cast<VTSDNode>(N1)->getVT();
11226 AssertVT == cast<VTSDNode>(N0.
getOperand(1))->getVT())
11237 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
11239 "Asserting zero/sign-extended bits to a type larger than the "
11240 "truncated destination does not provide information");
11243 EVT MinAssertVT = AssertVT.
bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
11257 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
11259 "Asserting zero/sign-extended bits to a type larger than the "
11260 "truncated destination does not provide information");
11262 if (AssertVT.
bitsLT(BigA_AssertVT)) {
11276 Align AL = cast<AssertAlignSDNode>(
N)->getAlign();
11281 if (
auto *AAN = dyn_cast<AssertAlignSDNode>(N0))
11293 unsigned AlignShift =
Log2(
AL);
11298 if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
11299 if (LHSAlignShift < AlignShift)
11301 if (RHSAlignShift < AlignShift)
11319 unsigned Opc =
N->getOpcode();
11323 EVT VT =
N->getValueType(0);
11330 unsigned ShAmt = 0;
11331 bool HasShiftedOffset =
false;
11336 ExtVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
11344 auto *LN0 = dyn_cast<LoadSDNode>(N0.
getOperand(0));
11345 auto *N01 = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
11349 uint64_t ShiftAmt = N01->getZExtValue();
11358 auto AndC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
11363 unsigned ActiveBits = 0;
11364 if (
Mask.isMask()) {
11365 ActiveBits =
Mask.countTrailingOnes();
11366 }
else if (
Mask.isShiftedMask()) {
11367 ShAmt =
Mask.countTrailingZeros();
11370 HasShiftedOffset =
true;
11380 if (
auto *ConstShift = dyn_cast<ConstantSDNode>(
SRL.getOperand(1))) {
11381 ShAmt = ConstShift->getZExtValue();
11384 if ((ShAmt & (EVTBits-1)) == 0) {
11392 auto *LN0 = dyn_cast<LoadSDNode>(N0);
11411 isa<ConstantSDNode>(
Mask->getOperand(1))) {
11412 const APInt& ShiftMask =
Mask->getConstantOperandAPInt(1);
11413 if (ShiftMask.
isMask()) {
11427 unsigned ShLeftAmt = 0;
11431 ShLeftAmt = N01->getZExtValue();
11437 if (!isa<LoadSDNode>(N0))
11444 !isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
11447 auto AdjustBigEndianShift = [&](
unsigned ShAmt) {
11448 unsigned LVTStoreBits =
11451 return LVTStoreBits - EVTStoreBits - ShAmt;
11457 ShAmt = AdjustBigEndianShift(ShAmt);
11459 uint64_t PtrOff = ShAmt / 8;
11467 AddToWorklist(NewPtr.
getNode());
11481 WorklistRemover DeadNodes(*
this);
11486 if (ShLeftAmt != 0) {
11501 if (HasShiftedOffset) {
11505 ShAmt = AdjustBigEndianShift(ShAmt);
11523 EVT VT =
N->getValueType(0);
11524 EVT ExtVT = cast<VTSDNode>(N1)->getVT();
11553 if ((N00Bits <= ExtVTBits ||
11564 if (!LegalOperations ||
11590 if (
SDValue NarrowLoad = ReduceLoadWidth(
N))
11597 if (
auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1)))
11598 if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) {
11602 if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
11614 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
11615 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple() &&
11623 CombineTo(
N, ExtLoad);
11625 AddToWorklist(ExtLoad.
getNode());
11631 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
11632 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) &&
11639 CombineTo(
N, ExtLoad);
11647 if (ExtVT == Ld->getMemoryVT() && N0.
hasOneUse() &&
11651 VT,
SDLoc(
N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
11652 Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
11653 Ld->getAddressingMode(),
ISD::SEXTLOAD, Ld->isExpandingLoad());
11654 CombineTo(
N, ExtMaskedLoad);
11661 if (
auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
11663 ExtVT == GN0->getMemoryVT() &&
11665 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
11666 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
11672 CombineTo(
N, ExtLoad);
11674 AddToWorklist(ExtLoad.
getNode());
11691 EVT VT =
N->getValueType(0);
11708 EVT VT =
N->getValueType(0);
11725 EVT VT =
N->getValueType(0);
11740 if (
C.getNode() !=
N)
11777 EVT TrTy =
N->getValueType(0);
11781 auto NewEltCnt = EltCnt * SizeRatio;
11787 if (isa<ConstantSDNode>(EltNo) &&
isTypeLegal(NVT)) {
11788 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
11789 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
11824 AddToWorklist(Amt.
getNode());
11858 if (BuildVectEltTy == TruncVecEltTy) {
11862 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
11864 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
11865 "Invalid number of elements");
11868 for (
unsigned i = 0,
e = BuildVecNumElts; i !=
e; i += TruncEltOffset)
11890 if (
SDValue Reduced = ReduceLoadWidth(
N))
11914 unsigned NumDefs = 0;
11918 if (!
X.isUndef()) {
11929 X.getValueType().getVectorElementCount()));
11935 if (NumDefs == 1) {
11936 assert(V.
getNode() &&
"The single defined operand is empty!");
11938 for (
unsigned i = 0,
e = VTs.
size(); i !=
e; ++i) {
11944 AddToWorklist(
NV.getNode());
11959 (!LegalOperations ||
12004 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
12018 if (!LegalOperations && N0.
hasOneUse() &&
12068 if (NewAlign <= Alignment &&
12087 EVT VT =
N->getValueType(0);
12127 NumFPLogicOpsConv++;
12138 EVT VT =
N->getValueType(0);
12154 cast<BuildVectorSDNode>(N0)->isConstant())
12155 return ConstantFoldBITCASTofBUILD_VECTOR(N0.
getNode(),
12159 if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
12163 if (!LegalOperations ||
12169 if (
C.getNode() !=
N)
12189 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) ||
12224 AddToWorklist(NewConv.
getNode());
12234 AddToWorklist(FlipBit.
getNode());
12241 AddToWorklist(
Hi.getNode());
12243 AddToWorklist(FlipBit.
getNode());
12247 AddToWorklist(FlipBits.
getNode());
12277 AddToWorklist(
X.getNode());
12281 if (OrigXWidth < VTWidth) {
12283 AddToWorklist(
X.getNode());
12284 }
else if (OrigXWidth > VTWidth) {
12289 X.getValueType(),
X,
12291 X.getValueType()));
12292 AddToWorklist(
X.getNode());
12294 AddToWorklist(
X.getNode());
12300 AddToWorklist(Cst.
getNode());
12302 AddToWorklist(
X.getNode());
12304 AddToWorklist(XorResult.
getNode());
12308 SDLoc(XorResult)));
12309 AddToWorklist(XorResult64.
getNode());
12313 AddToWorklist(FlipBit.
getNode());
12316 AddToWorklist(FlipBits.
getNode());
12322 AddToWorklist(
X.getNode());
12327 AddToWorklist(Cst.
getNode());
12335 if (
SDValue CombineLD = CombineConsecutiveLoads(N0.
getNode(), VT))
12350 auto PeekThroughBitcast = [&](
SDValue Op) {
12352 Op.getOperand(0).getValueType() == VT)
12372 for (
int i = 0; i != MaskScale; ++i)
12373 NewMask.
push_back(M < 0 ? -1 : M * MaskScale + i);
12378 return LegalShuffle;
12385 EVT VT =
N->getValueType(0);
12386 return CombineConsecutiveLoads(
N, VT);
12397 if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0))
12406 ConstantFoldBITCASTofBUILD_VECTOR(
SDNode *BV,
EVT DstEltVT) {
12410 if (SrcEltVT == DstEltVT)
return SDValue(BV, 0);
12417 if (SrcBitSize == DstBitSize) {
12422 if (
Op.getValueType() != SrcEltVT)
12425 AddToWorklist(Ops.
back().getNode());
12439 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
12447 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
12450 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
12458 if (SrcBitSize < DstBitSize) {
12459 unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
12463 i += NumInputsPerOutput) {
12466 bool EltIsUndef =
true;
12467 for (
unsigned j = 0; j != NumInputsPerOutput; ++j) {
12469 NewBits <<= SrcBitSize;
12471 if (
Op.isUndef())
continue;
12472 EltIsUndef =
false;
12474 NewBits |= cast<ConstantSDNode>(
Op)->getAPIntValue().
12475 zextOrTrunc(SrcBitSize).
zext(DstBitSize);
12490 unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
12496 if (
Op.isUndef()) {
12501 APInt OpVal = cast<ConstantSDNode>(
Op)->
12504 for (
unsigned j = 0; j != NumOutputsPerInput; ++j) {
12520 return F.hasAllowContract() ||
F.hasAllowReassociation();
12527 EVT VT =
N->getValueType(0);
12533 bool HasFMAD = (LegalOperations && TLI.
isFMADLegal(DAG,
N));
12541 if (!HasFMAD && !HasFMA)
12545 bool CanReassociate =
12546 Options.
UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
12548 CanFuse || HasFMAD);
12562 auto isContractableFMUL = [AllowFusionGlobally](
SDValue N) {
12569 if (
Aggressive && isContractableFMUL(N0) && isContractableFMUL(N1)) {
12591 if (CanReassociate && N0.
getOpcode() == PreferredFusedOpcode &&
12596 }
else if (CanReassociate && N1.
getOpcode() == PreferredFusedOpcode &&
12608 return DAG.
getNode(PreferredFusedOpcode, SL, VT, A,
B, CDE);
12616 if (isContractableFMUL(N00) &&
12619 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
12630 if (isContractableFMUL(N10) &&
12633 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
12646 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X,
Y,
12647 DAG.
getNode(PreferredFusedOpcode, SL, VT,
12652 if (N0.
getOpcode() == PreferredFusedOpcode) {
12656 if (isContractableFMUL(N020) &&
12676 DAG.
getNode(PreferredFusedOpcode, SL, VT,
12682 if (N00.
getOpcode() == PreferredFusedOpcode) {
12684 if (isContractableFMUL(N002) &&
12696 if (N1.
getOpcode() == PreferredFusedOpcode) {
12700 if (isContractableFMUL(N120) &&
12717 if (N10.
getOpcode() == PreferredFusedOpcode) {
12719 if (isContractableFMUL(N102) &&
12737 EVT VT =
N->getValueType(0);
12742 bool HasFMAD = (LegalOperations && TLI.
isFMADLegal(DAG,
N));
12750 if (!HasFMAD && !HasFMA)
12756 CanFuse || HasFMAD);
12772 auto isContractableFMUL = [AllowFusionGlobally](
SDValue N) {
12790 if (isContractableFMUL(YZ) && (
Aggressive || YZ->hasOneUse())) {
12791 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
12800 if (isContractableFMUL(N0) && isContractableFMUL(N1) &&
12803 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
12806 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
12810 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
12813 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
12822 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
12833 if (isContractableFMUL(N00) &&
12836 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
12848 if (isContractableFMUL(N10) &&
12852 PreferredFusedOpcode, SL, VT,
12869 if (isContractableFMUL(N000) &&
12874 DAG.
getNode(PreferredFusedOpcode, SL, VT,
12892 if (isContractableFMUL(N000) &&
12897 DAG.
getNode(PreferredFusedOpcode, SL, VT,
12909 if (CanFuse && N0.
getOpcode() == PreferredFusedOpcode &&
12914 DAG.
getNode(PreferredFusedOpcode, SL, VT,
12922 if (CanFuse && N1.
getOpcode() == PreferredFusedOpcode &&
12928 PreferredFusedOpcode, SL, VT,
12930 DAG.
getNode(PreferredFusedOpcode, SL, VT,
12937 if (N0.
getOpcode() == PreferredFusedOpcode &&
12942 if (isContractableFMUL(N020) &&
12948 PreferredFusedOpcode, SL, VT,
12964 if (N00.
getOpcode() == PreferredFusedOpcode) {
12966 if (isContractableFMUL(N002) &&
12970 PreferredFusedOpcode, SL, VT,
12974 PreferredFusedOpcode, SL, VT,
12984 if (N1.
getOpcode() == PreferredFusedOpcode &&
12988 if (isContractableFMUL(N120) &&
12994 PreferredFusedOpcode, SL, VT,
12996 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13015 if (isContractableFMUL(N102) &&
13021 PreferredFusedOpcode, SL, VT,
13025 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13039 SDValue DAGCombiner::visitFMULForFMADistributiveCombine(
SDNode *
N) {
13042 EVT VT =
N->getValueType(0);
13066 if (!HasFMAD && !HasFMA)
13078 if (
C->isExactlyValue(+1.0))
13079 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
13081 if (
C->isExactlyValue(-1.0))
13082 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
13101 if (C0->isExactlyValue(+1.0))
13102 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
13105 if (C0->isExactlyValue(-1.0))
13106 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
13111 if (C1->isExactlyValue(+1.0))
13112 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
13114 if (C1->isExactlyValue(-1.0))
13115 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
13135 EVT VT =
N->getValueType(0);
13146 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
13150 if (N0CFP && N1CFP)
13154 if (N0CFP && !N1CFP)
13159 if (N1C && N1C->
isZero())
13163 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
13169 N1, DAG, LegalOperations, ForCodeSize))
13175 N0, DAG, LegalOperations, ForCodeSize))
13182 return C &&
C->isExactlyValue(-2.0);
13186 if (isFMulNegTwo(N0)) {
13192 if (isFMulNegTwo(N1)) {
13235 if (CFP01 && !CFP00 && N0.
getOperand(0) == N1) {
13256 if (CFP11 && !CFP10 && N1.
getOperand(0) == N0) {
13304 if (
SDValue Fused = visitFADDForFMACombine(
N)) {
13305 AddToWorklist(Fused.getNode());
13315 EVT VT =
N->getValueType(0);
13316 EVT ChainVT =
N->getValueType(1);
13323 N1, DAG, LegalOperations, ForCodeSize)) {
13325 {Chain, N0, NegN1});
13331 N0, DAG, LegalOperations, ForCodeSize)) {
13333 {Chain, N1, NegN0});
13343 EVT VT =
N->getValueType(0);
13354 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
13358 if (N0CFP && N1CFP)
13361 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
13365 if (N1CFP && N1CFP->
isZero()) {
13379 if (N0CFP && N0CFP->
isZero()) {
13414 if (
SDValue Fused = visitFSUBForFMACombine(
N)) {
13415 AddToWorklist(Fused.getNode());
13427 EVT VT =
N->getValueType(0);
13439 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
13444 if (N0CFP && N1CFP)
13452 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
13498 if (NegN0 && NegN1 &&
13513 auto TrueOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(1));
13514 auto FalseOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(2));
13516 if (TrueOpnd && FalseOpnd &&
13518 isa<ConstantFPSDNode>(
Cond.getOperand(1)) &&
13519 cast<ConstantFPSDNode>(
Cond.getOperand(1))->isExactlyValue(0.0)) {
13537 if (TrueOpnd->isExactlyValue(-1.0) && FalseOpnd->isExactlyValue(1.0) &&
13541 if (TrueOpnd->isExactlyValue(1.0) && FalseOpnd->isExactlyValue(-1.0))
13550 if (
SDValue Fused = visitFMULForFMADistributiveCombine(
N)) {
13551 AddToWorklist(Fused.getNode());
13564 EVT VT =
N->getValueType(0);
13570 bool UnsafeFPMath =
13571 Options.
UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
13574 if (isa<ConstantFPSDNode>(N0) &&
13575 isa<ConstantFPSDNode>(N1) &&
13576 isa<ConstantFPSDNode>(N2)) {
13589 if (NegN0 && NegN1 &&
13594 if (UnsafeFPMath) {
13595 if (N0CFP && N0CFP->
isZero())
13597 if (N1CFP && N1CFP->
isZero())
13611 if (UnsafeFPMath) {
13638 AddToWorklist(RHSNeg.
getNode());
13652 if (UnsafeFPMath) {
13654 if (N1CFP && N0 == N2) {
13672 SDValue(
N, 0), DAG, LegalOperations, ForCodeSize))
13694 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
13705 unsigned NumElts = 1;
13706 EVT VT =
N->getValueType(0);
13710 if (!MinUses || (N1->
use_size() * NumElts) < MinUses)
13716 for (
auto *U : N1->
uses()) {
13717 if (U->getOpcode() ==
ISD::FDIV && U->getOperand(1) == N1) {
13719 if (U->getOperand(1).getOpcode() ==
ISD::FSQRT &&
13720 U->getOperand(0) == U->getOperand(1).getOperand(0) &&
13721 U->getFlags().hasAllowReassociation() &&
13722 U->getFlags().hasNoSignedZeros())
13727 if (UnsafeMath || U->getFlags().hasAllowReciprocal())
13734 if ((
Users.size() * NumElts) < MinUses)
13742 for (
auto *U :
Users) {
13744 if (Dividend != FPOne) {
13746 Reciprocal, Flags);
13747 CombineTo(U, NewNode);
13748 }
else if (U != Reciprocal.
getNode()) {
13751 CombineTo(U, Reciprocal);
13762 EVT VT =
N->getValueType(0);
13773 if (
SDValue FoldedVOp = SimplifyVBinOp(
N))
13777 if (N0CFP && N1CFP)
13780 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
13796 (!LegalOperations ||
13845 A =
Y.getOperand(0);
13854 if (
SDValue Rsqrt = buildRsqrtEstimate(AAZ, Flags))
13858 recursivelyDeleteUnusedNodes(AAZ.
getNode());
13866 AddToWorklist(Div.
getNode());
13874 if (
SDValue RV = BuildDivEstimate(N0, N1, Flags))
13893 if (NegN0 && NegN1 &&
13906 EVT VT =
N->getValueType(0);
13914 if (N0CFP && N1CFP)
13917 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
13942 return buildSqrtEstimate(N0, Flags);
13957 return (N1VT == N1Op0VT || N1Op0VT !=
MVT::f128);
13967 EVT VT =
N->getValueType(0);
13969 if (N0CFP && N1CFP)
14019 EVT VT =
N->getValueType(0);
14049 if (ExponentIs025 || ExponentIs075) {
14092 Attribute StrictOverflow =
F.getFnAttribute(
"strict-float-cast-overflow");
14102 EVT VT =
N->getValueType(0);
14123 EVT VT =
N->getValueType(0);
14133 (!LegalOperations ||
14175 EVT VT =
N->getValueType(0);
14185 (!LegalOperations ||
14215 EVT VT =
N->getValueType(0);
14236 unsigned ActualSize =
std::min(InputSize, OutputSize);
14256 EVT VT =
N->getValueType(0);
14271 EVT VT =
N->getValueType(0);
14288 EVT VT =
N->getValueType(0);
14300 const bool NIsTrunc =
N->getConstantOperandVal(1) == 1;
14329 AddToWorklist(Tmp.
getNode());
14334 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
14342 EVT VT =
N->getValueType(0);
14345 if (
N->hasOneUse() &&
14363 if (
In.getValueType() == VT)
return In;
14378 CombineTo(
N, ExtLoad);
14387 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
14395 EVT VT =
N->getValueType(0);
14406 EVT VT =
N->getValueType(0);
14430 EVT VT =
N->getValueType(0);
14441 EVT VT =
N->getValueType(0);
14458 N->getFlags().hasNoSignedZeros()) && N0.
hasOneUse()) {
14463 if (
SDValue Cast = foldSignChangeInBitcast(
N))
14473 EVT VT =
N->getValueType(0);
14477 unsigned Opc =
N->getOpcode();
14482 if (N0CFP && N1CFP) {
14501 return PropagatesNaN ?
N->getOperand(1) :
N->getOperand(0);
14511 return N->getOperand(1);
14518 return N->getOperand(0);
14543 EVT VT =
N->getValueType(0);
14551 return N->getOperand(0);
14558 if (
SDValue Cast = foldSignChangeInBitcast(
N))
14596 if (
SDValue NewN1 = rebuildSetCC(N1))
14598 ChainHandle.getValue(), NewN1, N2);
14607 (
N.getOperand(0).hasOneUse() &&
14608 N.getOperand(0).getOpcode() ==
ISD::SRL))) {
14611 N =
N.getOperand(0);
14637 const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue();
14640 cast<ConstantSDNode>(Op1)->getAPIntValue() == AndConst.
logBase2()) {
14660 SDValue Tmp = visitXOR(
N.getNode());
14667 N = XORHandle.getValue();
14679 bool Equal =
false;
14684 Op0 =
N->getOperand(0);
14685 Op1 =
N->getOperand(1);
14689 EVT SetCCVT =
N.getValueType();
14705 SDValue CondLHS =
N->getOperand(2), CondRHS =
N->getOperand(3);
14730 bool &IsLoad,
bool &IsMasked,
SDValue &Ptr,
14733 if (
LD->isIndexed())
14735 EVT VT =
LD->getMemoryVT();
14738 Ptr =
LD->getBasePtr();
14740 if (
ST->isIndexed())
14742 EVT VT =
ST->getMemoryVT();
14745 Ptr =
ST->getBasePtr();
14748 if (
LD->isIndexed())
14750 EVT VT =
LD->getMemoryVT();
14754 Ptr =
LD->getBasePtr();
14757 if (
ST->isIndexed())
14759 EVT VT =
ST->getMemoryVT();
14763 Ptr =
ST->getBasePtr();
14777 bool DAGCombiner::CombineToPreIndexedLoadStore(
SDNode *
N) {
14781 bool IsLoad =
true;
14782 bool IsMasked =
false;
14804 bool Swapped =
false;
14805 if (isa<ConstantSDNode>(BasePtr)) {
14824 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
14829 SDValue Val = IsMasked ? cast<MaskedStoreSDNode>(
N)->getValue()
14830 : cast<StoreSDNode>(
N)->getValue();
14833 if (Val == BasePtr)
14850 if (isa<ConstantSDNode>(
Offset))
14852 UE =
BasePtr.getNode()->use_end();
14870 if (!isa<ConstantSDNode>(Op1)) {
14888 bool RealUse =
false;
14923 Result.getNode()->dump(&DAG);
dbgs() <<
'\n');
14924 WorklistRemover DeadNodes(*
this);
14933 deleteAndRecombine(
N);
14939 for (
unsigned i = 0,
e = OtherUses.
size(); i !=
e; ++i) {
14940 unsigned OffsetIdx = 1;
14941 if (OtherUses[i]->getOperand(OffsetIdx).getNode() ==
BasePtr.getNode())
14943 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() ==
14944 BasePtr.getNode() &&
"Expected BasePtr operand");
14957 auto *CN = cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx));
14958 const APInt &Offset0 = CN->getAPIntValue();
14959 const APInt &Offset1 = cast<ConstantSDNode>(
Offset)->getAPIntValue();
14960 int X0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 1) ? -1 : 1;
14961 int Y0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 0) ? -1 : 1;
14967 APInt CNV = Offset0;
14968 if (X0 < 0) CNV = -CNV;
14969 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
14970 else CNV = CNV - Offset1;
14982 deleteAndRecombine(OtherUses[i]);
14987 deleteAndRecombine(Ptr.
getNode());
14988 AddToWorklist(
Result.getNode());
15009 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
15013 for (
SDNode *
Use : BasePtr.getNode()->uses()) {
15018 if (isa<MemSDNode>(
Use)) {
15019 bool IsLoad =
true;
15020 bool IsMasked =
false;
15023 IsMasked, OtherPtr, TLI)) {
15043 bool &IsMasked,
SDValue &Ptr,
15049 IsMasked, Ptr, TLI) ||
15082 bool DAGCombiner::CombineToPostIndexedLoadStore(
SDNode *
N) {
15086 bool IsLoad =
true;
15087 bool IsMasked =
false;
15108 ++PostIndexedNodes;
15111 dbgs() <<
"\nWith: ";
Result.getNode()->dump(&DAG);
15113 WorklistRemover DeadNodes(*
this);
15122 deleteAndRecombine(
N);
15126 Result.getValue(IsLoad ? 1 : 0));
15127 deleteAndRecombine(
Op);
15142 !cast<ConstantSDNode>(Inc)->isOpaque()) &&
15143 "Cannot split out indexing using opaque target constants");
15160 Val =
ST->getValue();
15162 EVT STMemType =
ST->getMemoryVT();
15163 if (STType == STMemType)
15185 EVT LDMemType =
LD->getMemoryVT();
15186 EVT LDType =
LD->getValueType(0);
15188 "Attempting to extend value of non-matching type");
15189 if (LDType == LDMemType)
15192 switch (
LD->getExtensionType()) {
15216 if (!
ST || !
ST->isSimple())
15219 EVT LDType =
LD->getValueType(0);
15220 EVT LDMemType =
LD->getMemoryVT();
15221 EVT STMemType =
ST->getMemoryVT();
15222 EVT STType =
ST->getValue().getValueType();
15264 STCoversLD = (
Offset == 0) && LdMemSize == StMemSize;
15270 if (
LD->isIndexed()) {
15275 SDValue Idx = SplitIndexingFromLoad(
LD);
15276 SDValue Ops[] = {Val, Idx, Chain};
15277 return CombineTo(
LD, Ops, 3);
15279 return CombineTo(
LD, Val, Chain);
15286 if (
Offset == 0 && LDType == STType && STMemType == LDMemType) {
15289 return ReplaceLd(
LD,
ST->getValue(), Chain);
15299 return ReplaceLd(
LD, Val, Chain);
15304 if (
LD->getBasePtr().isUndef() ||
Offset != 0)
15310 if (!getTruncatedStoreValue(
ST, Val))
15314 if (STMemType != LDMemType) {
15322 if (!extendLoadedValueToExtension(
LD, Val))
15324 return ReplaceLd(
LD, Val, Chain);
15329 deleteAndRecombine(Val.
getNode());
15342 if (
LD->isSimple()) {
15345 if (!
N->hasAnyUseOfValue(0)) {
15355 WorklistRemover DeadNodes(*
this);
15357 AddUsersToWorklist(Chain.
getNode());
15358 if (
N->use_empty())
15359 deleteAndRecombine(
N);
15373 if (!
N->hasAnyUseOfValue(0) && (CanSplitIdx || !
N->hasAnyUseOfValue(1))) {
15376 if (
N->hasAnyUseOfValue(1) && CanSplitIdx) {
15377 Index = SplitIndexingFromLoad(
LD);
15380 AddUsersToWorklist(
N);
15384 dbgs() <<
"\nWith: ";
Undef.getNode()->dump(&DAG);
15385 dbgs() <<
" and 2 other values\n");
15386 WorklistRemover DeadNodes(*
this);
15390 deleteAndRecombine(
N);
15398 if (
auto V = ForwardStoreValueToDirectLoad(
LD))
15404 if (*Alignment >
LD->getAlign() &&
15405 isAligned(*Alignment,
LD->getSrcValueOffset())) {
15407 LD->getExtensionType(),
SDLoc(
N),
LD->getValueType(0), Chain, Ptr,
15408 LD->getPointerInfo(),
LD->getMemoryVT(), *Alignment,
15409 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
15417 if (
LD->isUnindexed()) {
15419 SDValue BetterChain = FindBetterChain(
LD, Chain);
15422 if (Chain != BetterChain) {
15428 BetterChain, Ptr,
LD->getMemOperand());
15431 LD->getValueType(0),
15432 BetterChain, Ptr,
LD->getMemoryVT(),
15433 LD->getMemOperand());
15441 return CombineTo(
N, ReplLoad.
getValue(0), Token);
15446 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
15451 if (SliceUpLoad(
N))
15471 struct LoadedSlice {
15475 bool ForCodeSize =
false;
15478 unsigned Loads = 0;
15479 unsigned Truncates = 0;
15480 unsigned CrossRegisterBanksCopies = 0;
15481 unsigned ZExts = 0;
15482 unsigned Shift = 0;
15484 explicit Cost(
bool ForCodeSize) : ForCodeSize(ForCodeSize) {}
15487 Cost(
const LoadedSlice &
LS,
bool ForCodeSize)
15488 : ForCodeSize(ForCodeSize), Loads(1) {
15489 EVT TruncType =
LS.Inst->getValueType(0);
15490 EVT LoadedType =
LS.getLoadedType();
15491 if (TruncType != LoadedType &&
15492 !
LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
15500 void addSliceGain(
const LoadedSlice &
LS) {
15504 LS.Inst->getValueType(0)))
15510 if (
LS.canMergeExpensiveCrossRegisterBankCopy())
15511 ++CrossRegisterBanksCopies;
15515 Loads += RHS.Loads;
15516 Truncates += RHS.Truncates;
15517 CrossRegisterBanksCopies += RHS.CrossRegisterBanksCopies;
15518 ZExts += RHS.ZExts;
15519 Shift += RHS.Shift;
15524 return Loads == RHS.Loads && Truncates == RHS.Truncates &&
15525 CrossRegisterBanksCopies == RHS.CrossRegisterBanksCopies &&
15526 ZExts == RHS.ZExts && Shift == RHS.Shift;
15529 bool operator!=(
const Cost &RHS)
const {
return !(*
this == RHS); }
15531 bool operator<(
const Cost &RHS)
const {
15534 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
15535 unsigned ExpensiveOpsRHS = RHS.Loads + RHS.CrossRegisterBanksCopies;
15538 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
15539 return ExpensiveOpsLHS < ExpensiveOpsRHS;
15540 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
15541 (RHS.Truncates + RHS.ZExts + RHS.Shift + ExpensiveOpsRHS);
15544 bool operator>(
const Cost &RHS)
const {
return RHS < *
this; }
15546 bool operator<=(
const Cost &RHS)
const {
return !(RHS < *
this); }
15548 bool operator>=(
const Cost &RHS)
const {
return !(*
this < RHS); }
15567 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
15572 APInt getUsedBits()
const {
15577 assert(Origin &&
"No original load to compare against.");
15579 assert(Inst &&
"This slice is not bound to an instruction");
15581 "Extracted slice is bigger than the whole type!");
15583 UsedBits.setAllBits();
15584 UsedBits = UsedBits.zext(
BitWidth);
15585 UsedBits <<= Shift;
15590 unsigned getLoadedSize()
const {
15591 unsigned SliceSize = getUsedBits().countPopulation();
15592 assert(!(SliceSize & 0x7) &&
"Size is not a multiple of a byte.");
15593 return SliceSize / 8;
15598 EVT getLoadedType()
const {
15599 assert(DAG &&
"Missing context");
15607 uint64_t
Offset = getOffsetFromBase();
15614 bool isLegal()
const {
15616 if (!Origin || !Inst || !DAG)
15626 EVT SliceType = getLoadedType();
15650 if (TruncateType != SliceType &&
15660 uint64_t getOffsetFromBase()
const {
15661 assert(DAG &&
"Missing context.");
15663 assert(!(Shift & 0x7) &&
"Shifts not aligned on Bytes are not supported.");
15664 uint64_t
Offset = Shift / 8;
15667 "The size of the original loaded type is not a multiple of a"
15672 "Invalid shift amount for given loaded size");
15685 assert(Inst && Origin &&
"Unable to replace a non-existing slice.");
15687 SDValue BaseAddr = OldBaseAddr;
15689 int64_t
Offset =
static_cast<int64_t
>(getOffsetFromBase());
15690 assert(
Offset >= 0 &&
"Offset too big to fit in int64_t!");
15700 EVT SliceType = getLoadedType();
15710 if (SliceType != FinalType)
15720 bool canMergeExpensiveCrossRegisterBankCopy()
const {
15726 assert(DAG &&
"Missing context");
15728 EVT ResVT =
Use->getValueType(0);
15733 Use->getOperand(0)->isDivergent());
15742 if (!
TRI ||
TRI->getCommonSubClass(ArgRC, ResRC))
15750 if (RequiredAlignment >
getAlign())
15787 const LoadedSlice &Second) {
15788 assert(First.Origin == Second.Origin && First.Origin &&
15789 "Unable to match different memory origins.");
15790 APInt UsedBits = First.getUsedBits();
15791 assert((UsedBits & Second.getUsedBits()) == 0 &&
15792 "Slices are not supposed to overlap.");
15793 UsedBits |= Second.getUsedBits();
15802 LoadedSlice::Cost &GlobalLSCost) {
15803 unsigned NumberOfSlices = LoadedSlices.
size();
15805 if (NumberOfSlices < 2)
15810 llvm::sort(LoadedSlices, [](
const LoadedSlice &LHS,
const LoadedSlice &RHS) {
15811 assert(LHS.Origin == RHS.Origin &&
"Different bases not implemented.");
15812 return LHS.getOffsetFromBase() < RHS.getOffsetFromBase();
15814 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
15817 const LoadedSlice *First =
nullptr;
15818 const LoadedSlice *Second =
nullptr;
15819 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
15822 Second = &LoadedSlices[CurrSlice];
15829 EVT LoadedType = First->getLoadedType();
15832 if (LoadedType != Second->getLoadedType())
15836 Align RequiredAlignment;
15837 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
15843 if (First->getAlign() < RequiredAlignment)
15850 assert(GlobalLSCost.Loads > 0 &&
"We save more loads than we created!");
15851 --GlobalLSCost.Loads;
15868 const APInt &UsedBits,
bool ForCodeSize) {
15869 unsigned NumberOfSlices = LoadedSlices.
size();
15871 return NumberOfSlices > 1;
15874 if (NumberOfSlices != 2)
15882 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
15884 OrigCost.Loads = 1;
15885 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
15886 const LoadedSlice &
LS = LoadedSlices[CurrSlice];
15888 LoadedSlice::Cost SliceCost(
LS, ForCodeSize);
15889 GlobalSlicingCost += SliceCost;
15893 OrigCost.addSliceGain(
LS);
15898 return OrigCost > GlobalSlicingCost;
15907 bool DAGCombiner::SliceUpLoad(
SDNode *
N) {
15913 !
LD->getValueType(0).isInteger())
15919 if (
LD->getValueType(0).isScalableVector())
15924 APInt UsedBits(
LD->getValueSizeInBits(0), 0);
15932 UI != UIEnd; ++UI) {
15934 if (UI.getUse().getResNo() != 0)
15938 unsigned Shift = 0;
15943 Shift =
User->getConstantOperandVal(1);
15956 unsigned Width =
User->getValueSizeInBits(0);
15961 LoadedSlice
LS(
User,
LD, Shift, &DAG);
15962 APInt CurrentUsedBits =
LS.getUsedBits();
15965 if ((CurrentUsedBits & UsedBits) != 0)
15968 UsedBits |= CurrentUsedBits;
15990 LSIt = LoadedSlices.
begin(),
15991 LSItEnd = LoadedSlices.
end();
15992 LSIt != LSItEnd; ++LSIt) {
15993 SDValue SliceInst = LSIt->loadSlice();
15994 CombineTo(LSIt->Inst, SliceInst,
true);
15998 "It takes more than a zext to get to the loaded slice!!");
16005 AddToWorklist(Chain.
getNode());
16012 static std::pair<unsigned, unsigned>
16014 std::pair<unsigned, unsigned> Result(0, 0);
16024 if (
LD->getBasePtr() != Ptr)
return Result;
16035 uint64_t NotMask = ~cast<ConstantSDNode>(V->
getOperand(1))->getSExtValue();
16037 if (NotMaskLZ & 7)
return Result;
16039 if (NotMaskTZ & 7)
return Result;
16040 if (NotMaskLZ == 64)
return Result;
16051 switch (MaskedBytes) {
16055 default:
return Result;
16060 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes)
return Result;
16074 Result.first = MaskedBytes;
16075 Result.second = NotMaskTZ/8;
16086 unsigned NumBytes = MaskInfo.first;
16087 unsigned ByteShift = MaskInfo.second;
16093 ByteShift*8, (ByteShift+NumBytes)*8);
16101 if (!
DC->isTypeLegal(VT))
16120 StOffset = ByteShift;
16146 if (!
ST->isSimple())
16157 unsigned Opc =
Value.getOpcode();
16165 std::pair<unsigned, unsigned> MaskedLoad;
16167 if (MaskedLoad.first)
16169 Value.getOperand(1),
ST,
this))
16174 if (MaskedLoad.first)
16176 Value.getOperand(0),
ST,
this))
16191 if (
LD->getBasePtr() != Ptr ||
16192 LD->getPointerInfo().getAddrSpace() !=
16193 ST->getPointerInfo().getAddrSpace())
16199 APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
16223 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
16226 if ((Imm &
Mask) == Imm) {
16230 uint64_t PtrOff = ShAmt / 8;
16234 PtrOff = (
BitWidth + 7 - NewBW) / 8 - PtrOff;
16245 LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
16246 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
16252 ST->getPointerInfo().getWithOffset(PtrOff), NewAlign);
16254 AddToWorklist(NewPtr.
getNode());
16255 AddToWorklist(NewLD.
getNode());
16256 AddToWorklist(NewVal.
getNode());
16257 WorklistRemover DeadNodes(*
this);
16276 EVT VT =
LD->getMemoryVT();
16278 VT !=
ST->getMemoryVT() ||
16279 LD->isNonTemporal() ||
16280 ST->isNonTemporal() ||
16281 LD->getPointerInfo().getAddrSpace() != 0 ||
16282 ST->getPointerInfo().getAddrSpace() != 0)
16299 Align LDAlign =
LD->getAlign();
16300 Align STAlign =
ST->getAlign();
16303 if (LDAlign < ABIAlign || STAlign < ABIAlign)
16308 LD->getPointerInfo(), LDAlign);
16312 ST->getPointerInfo(), STAlign);
16314 AddToWorklist(NewLD.
getNode());
16315 AddToWorklist(NewST.
getNode());
16316 WorklistRemover DeadNodes(*
this);
16338 bool DAGCombiner::isMulAddWithConstProfitable(
SDNode *MulNode,
16349 if (
Use == MulNode)
16357 if (
Use->getOperand(0) == ConstNode)
16358 OtherOp =
Use->getOperand(1).getNode();
16360 OtherOp =
Use->getOperand(0).getNode();
16372 if (OtherOp == MulVar)
16400 unsigned NumStores) {
16403 SDLoc StoreDL(StoreNodes[0].MemNode);
16405 for (
unsigned i = 0; i < NumStores; ++i) {
16406 Visited.
insert(StoreNodes[i].MemNode);
16410 for (
unsigned i = 0; i < NumStores; ++i) {
16411 if (Visited.
insert(StoreNodes[i].MemNode->getChain().getNode()).second)
16412 Chains.
push_back(StoreNodes[i].MemNode->getChain());
16415 assert(Chains.
size() > 0 &&
"Chain should have generated a chain");
16419 bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
16421 bool IsConstantSrc,
bool UseVector,
bool UseTrunc) {
16427 SDLoc DL(StoreNodes[0].MemNode);
16430 unsigned SizeInBits = NumStores * ElementSizeBits;
16435 unsigned Elts = NumStores * NumMemElts;
16443 if (IsConstantSrc) {
16445 for (
unsigned I = 0;
I != NumStores; ++
I) {
16446 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
I].MemNode);
16455 if (isa<ConstantFPSDNode>(Val)) {
16458 }
else if (
auto *
C = dyn_cast<ConstantSDNode>(Val))
16461 .zextOrTrunc(ElementSizeBits),
16471 DL, StoreTy, BuildVector);
16474 for (
unsigned i = 0; i < NumStores; ++i) {
16475 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
16509 assert(IsConstantSrc &&
"Merged vector elements should use vector store");
16511 APInt StoreInt(SizeInBits, 0);
16516 for (
unsigned i = 0; i < NumStores; ++i) {
16517 unsigned Idx = IsLE ? (NumStores - 1 - i) : i;
16518 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[Idx].MemNode);
16520 SDValue Val = St->getValue();
16522 StoreInt <<= ElementSizeBits;
16524 StoreInt |=
C->getAPIntValue()
16525 .zextOrTrunc(ElementSizeBits)
16526 .zextOrTrunc(SizeInBits);
16528 StoreInt |=
C->getValueAPF()
16530 .zextOrTrunc(ElementSizeBits)
16531 .zextOrTrunc(SizeInBits);
16545 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
16554 EVT LegalizedStoredValTy =
16556 unsigned LegalizedStoreSize = LegalizedStoredValTy.
getSizeInBits();
16559 DAG.
getConstant(
C->getAPIntValue().zextOrTrunc(LegalizedStoreSize),
DL,
16560 LegalizedStoredValTy);
16562 NewChain,
DL, ExtendedStoreVal, FirstInChain->
getBasePtr(),
16568 for (
unsigned i = 0; i < NumStores; ++i)
16569 CombineTo(StoreNodes[i].MemNode, NewStore);
16571 AddToWorklist(NewChain.
getNode());
16575 void DAGCombiner::getStoreMergeCandidates(
16582 if (!
BasePtr.getBase().getNode() ||
BasePtr.getBase().isUndef())
16586 StoreSource StoreSrc = getStoreSource(Val);
16594 auto *Ld = cast<LoadSDNode>(Val);
16596 LoadVT = Ld->getMemoryVT();
16598 if (MemVT != LoadVT)
16601 if (!Ld->hasNUsesOfValue(1, 0))
16605 if (!Ld->isSimple() || Ld->isIndexed())
16609 int64_t &
Offset) ->
bool {
16612 if (!
Other->isSimple() ||
Other->isIndexed())
16620 :
Other->getMemoryVT() != MemVT;
16621 switch (StoreSrc) {
16626 auto *OtherLd = dyn_cast<LoadSDNode>(OtherBC);
16630 if (LoadVT != OtherLd->getMemoryVT())
16633 if (!OtherLd->hasNUsesOfValue(1, 0))
16637 if (!OtherLd->isSimple() || OtherLd->isIndexed())
16640 if (cast<LoadSDNode>(Val)->isNonTemporal() != OtherLd->isNonTemporal())
16649 if (!(isa<ConstantSDNode>(OtherBC) || isa<ConstantFPSDNode>(OtherBC)))
16652 case StoreSource::Extract:
16654 if (
Other->isTruncatingStore())
16671 auto OverLimitInDependenceCheck = [&](
SDNode *StoreNode,
16672 SDNode *RootNode) ->
bool {
16673 auto RootCount = StoreRootCountMap.
find(StoreNode);
16674 return RootCount != StoreRootCountMap.
end() &&
16675 RootCount->second.first == RootNode &&
16681 if (UseIter.getOperandNo() != 0)
16683 if (
auto *OtherStore = dyn_cast<StoreSDNode>(*UseIter)) {
16686 if (CandidateMatch(OtherStore, Ptr, PtrDiff) &&
16687 !OverLimitInDependenceCheck(OtherStore, RootNode))
16688 StoreNodes.
push_back(MemOpLink(OtherStore, PtrDiff));
16710 unsigned NumNodesExplored = 0;
16711 const unsigned MaxSearchNodes = 1024;
16712 if (
auto *Ldn = dyn_cast<LoadSDNode>(RootNode)) {
16713 RootNode = Ldn->getChain().getNode();
16715 I !=
E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored) {
16716 if (
I.getOperandNo() == 0 && isa<LoadSDNode>(*
I)) {
16717 for (
auto I2 = (*I)->use_begin(), E2 = (*I)->use_end(); I2 != E2; ++I2)
16718 TryToAddCandidate(I2);
16723 I !=
E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored)
16724 TryToAddCandidate(
I);
16733 bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
16749 while (!Worklist.
empty()) {
16751 if (!Visited.
insert(
N).second)
16760 unsigned int Max = 1024 + Visited.
size();
16762 for (
unsigned i = 0; i < NumStores; ++i) {
16763 SDNode *
N = StoreNodes[i].MemNode;
16775 for (
unsigned j = 1; j <
N->getNumOperands(); ++j)
16776 Worklist.
push_back(
N->getOperand(j).getNode());
16779 for (
unsigned i = 0; i < NumStores; ++i)
16785 if (Visited.
size() >= Max) {
16786 auto &RootCount = StoreRootCountMap[StoreNodes[i].MemNode];
16787 if (RootCount.first == RootNode)
16788 RootCount.second++;
16790 RootCount = {RootNode, 1};
16799 int64_t ElementSizeBytes)
const {
16802 size_t StartIdx = 0;
16803 while ((StartIdx + 1 < StoreNodes.
size()) &&
16804 StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
16805 StoreNodes[StartIdx + 1].OffsetFromBase)
16809 if (StartIdx + 1 >= StoreNodes.
size())
16818 unsigned NumConsecutiveStores = 1;
16819 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
16822 for (
unsigned i = 1,
e = StoreNodes.
size(); i <
e; ++i) {
16823 int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
16824 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
16826 NumConsecutiveStores = i + 1;
16828 if (NumConsecutiveStores > 1)
16829 return NumConsecutiveStores;
16837 bool DAGCombiner::tryStoreMergeOfConstants(
16839 EVT MemVT,
SDNode *RootNode,
bool AllowVectors) {
16844 bool MadeChange =
false;
16847 while (NumConsecutiveStores >= 2) {
16850 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
16851 unsigned LastLegalType = 1;
16852 unsigned LastLegalVectorType = 1;
16853 bool LastIntegerTrunc =
false;
16854 bool NonZero =
false;
16855 unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
16856 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
16857 StoreSDNode *
ST = cast<StoreSDNode>(StoreNodes[i].MemNode);
16859 bool IsElementZero =
false;
16861 IsElementZero =
C->isNullValue();
16863 IsElementZero =
C->getConstantFPValue()->isNullValue();
16864 if (IsElementZero) {
16865 if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
16866 FirstZeroAfterNonZero = i;
16868 NonZero |= !IsElementZero;
16871 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
16873 bool IsFast =
false;
16884 LastIntegerTrunc =
false;
16885 LastLegalType = i + 1;
16889 EVT LegalizedStoredValTy =
16896 LastIntegerTrunc =
true;
16897 LastLegalType = i + 1;
16908 unsigned Elts = (i + 1) * NumMemElts;
16915 LastLegalVectorType = i + 1;
16919 bool UseVector = (LastLegalVectorType > LastLegalType) && AllowVectors;
16920 unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
16932 unsigned NumSkip = 1;
16933 while ((NumSkip < NumConsecutiveStores) &&
16934 (NumSkip < FirstZeroAfterNonZero) &&
16935 (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
16939 NumConsecutiveStores -= NumSkip;
16944 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
16947 NumConsecutiveStores -= NumElem;
16951 MadeChange |= mergeStoresOfConstantsOrVecElts(
16952 StoreNodes, MemVT, NumElem,
true, UseVector, LastIntegerTrunc);
16956 NumConsecutiveStores -= NumElem;
16961 bool DAGCombiner::tryStoreMergeOfExtracts(
16967 bool MadeChange =
false;
16970 while (NumConsecutiveStores >= 2) {
16973 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
16974 unsigned NumStoresToMerge = 1;
16975 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
16977 unsigned Elts = (i + 1) * NumMemElts;
16979 bool IsFast =
false;
16989 NumStoresToMerge = i + 1;
16994 if (NumStoresToMerge < 2) {
17001 unsigned NumSkip = 1;
17002 while ((NumSkip < NumConsecutiveStores) &&
17003 (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
17007 NumConsecutiveStores -= NumSkip;
17012 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStoresToMerge,
17015 StoreNodes.
begin() + NumStoresToMerge);
17016 NumConsecutiveStores -= NumStoresToMerge;
17020 MadeChange |= mergeStoresOfConstantsOrVecElts(
17021 StoreNodes, MemVT, NumStoresToMerge,
false,
true,
false);
17023 StoreNodes.
erase(StoreNodes.
begin(), StoreNodes.
begin() + NumStoresToMerge);
17024 NumConsecutiveStores -= NumStoresToMerge;
17030 unsigned NumConsecutiveStores,
EVT MemVT,
17031 SDNode *RootNode,
bool AllowVectors,
17032 bool IsNonTemporalStore,
17033 bool IsNonTemporalLoad) {
17038 bool MadeChange =
false;
17040 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
17049 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
17050 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
17056 int64_t LdOffset = 0;
17067 LoadNodes.
push_back(MemOpLink(Ld, LdOffset));
17070 while (NumConsecutiveStores >= 2 && LoadNodes.
size() >= 2) {
17071 Align RequiredAlignment;
17072 bool NeedRotate =
false;
17073 if (LoadNodes.
size() == 2) {
17077 StoreNodes[0].MemNode->getAlign() >= RequiredAlignment) {
17083 int64_t Offset0 = LoadNodes[0].OffsetFromBase;
17084 int64_t Offset1 = LoadNodes[1].OffsetFromBase;
17086 if (Offset0 - Offset1 == ElementSizeBytes &&
17096 LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
17102 unsigned LastConsecutiveLoad = 1;
17105 unsigned LastLegalVectorType = 1;
17106 unsigned LastLegalIntegerType = 1;
17107 bool isDereferenceable =
true;
17108 bool DoIntegerTruncate =
false;
17109 StartAddress = LoadNodes[0].OffsetFromBase;
17111 for (
unsigned i = 1; i < LoadNodes.
size(); ++i) {
17113 if (LoadNodes[i].MemNode->getChain() != LoadChain)
17116 int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
17117 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
17119 LastConsecutiveLoad = i;
17121 if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
17122 isDereferenceable =
false;
17125 unsigned Elts = (i + 1) * NumMemElts;
17132 bool IsFastSt =
false;
17133 bool IsFastLd =
false;
17142 LastLegalVectorType = i + 1;
17146 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
17156 LastLegalIntegerType = i + 1;
17157 DoIntegerTruncate =
false;
17173 LastLegalIntegerType = i + 1;
17174 DoIntegerTruncate =
true;
17182 LastLegalVectorType > LastLegalIntegerType && AllowVectors;
17183 unsigned LastLegalType =
17184 std::max(LastLegalVectorType, LastLegalIntegerType);
17188 unsigned NumElem =
std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
17189 NumElem =
std::min(LastLegalType, NumElem);
17200 unsigned NumSkip = 1;
17201 while ((NumSkip < LoadNodes.
size()) &&
17202 (LoadNodes[NumSkip].MemNode->getAlign() <= FirstLoadAlign) &&
17203 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
17207 NumConsecutiveStores -= NumSkip;
17212 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
17216 NumConsecutiveStores -= NumElem;
17225 unsigned Elts = NumElem * NumMemElts;
17228 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
17232 SDLoc LoadDL(LoadNodes[0].MemNode);
17233 SDLoc StoreDL(StoreNodes[0].MemNode);
17238 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
17239 AddToWorklist(NewStoreChain.
getNode());
17244 if (IsNonTemporalLoad)
17252 if (UseVectorTy || !DoIntegerTruncate) {
17258 unsigned LoadWidth = ElementSizeBytes * 8 * 2;
17260 "Unexpected type for rotate-able load pair");
17267 NewStoreChain, StoreDL, StoreOp, FirstInChain->
getBasePtr(),
17275 FirstLoadAlign, LdMMOFlags);
17277 NewStoreChain, StoreDL, NewLoad, FirstInChain->
getBasePtr(),
17283 for (
unsigned i = 0; i < NumElem; ++i) {
17284 LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
17291 for (
unsigned i = 0; i < NumElem; ++i) {
17292 SDValue Val = StoreNodes[i].MemNode->getOperand(1);
17293 CombineTo(StoreNodes[i].MemNode, NewStore);
17295 recursivelyDeleteUnusedNodes(Val.
getNode());
17301 NumConsecutiveStores -= NumElem;
17306 bool DAGCombiner::mergeConsecutiveStores(
StoreSDNode *St) {
17322 if (ElementSizeBytes * 8 != (int64_t)MemVT.
getSizeInBits())
17328 const StoreSource StoreSrc = getStoreSource(StoredVal);
17335 getStoreMergeCandidates(St, StoreNodes, RootNode);
17338 if (StoreNodes.
size() < 2)
17343 llvm::sort(StoreNodes, [](MemOpLink LHS, MemOpLink RHS) {
17344 return LHS.OffsetFromBase < RHS.OffsetFromBase;
17348 Attribute::NoImplicitFloat);
17351 cast<LoadSDNode>(StoredVal)->isNonTemporal();
17360 bool MadeChange =
false;
17361 while (StoreNodes.
size() > 1) {
17362 unsigned NumConsecutiveStores =
17363 getConsecutiveStores(StoreNodes, ElementSizeBytes);
17365 if (NumConsecutiveStores == 0)
17369 assert(NumConsecutiveStores >= 2 &&
"Expected at least 2 stores");
17370 switch (StoreSrc) {
17372 MadeChange |= tryStoreMergeOfConstants(StoreNodes, NumConsecutiveStores,
17373 MemVT, RootNode, AllowVectors);
17376 case StoreSource::Extract:
17377 MadeChange |= tryStoreMergeOfExtracts(StoreNodes, NumConsecutiveStores,
17382 MadeChange |= tryStoreMergeOfLoads(StoreNodes, NumConsecutiveStores,
17383 MemVT, RootNode, AllowVectors,
17384 IsNonTemporalStore, IsNonTemporalLoad);
17399 if (
ST->isTruncatingStore()) {
17401 ST->getBasePtr(),
ST->getMemoryVT(),
17402 ST->getMemOperand());
17404 ReplStore = DAG.
getStore(BetterChain, SL,
ST->getValue(),
ST->getBasePtr(),
17405 ST->getMemOperand());
17413 AddToWorklist(Token.
getNode());
17416 return CombineTo(
ST, Token,
false);
17453 bitcastToAPInt().getZExtValue(),
SDLoc(CFP),
17455 return DAG.
getStore(Chain,
DL, Tmp, Ptr,
ST->getMemOperand());
17467 Ptr,
ST->getMemOperand());
17470 if (
ST->isSimple() &&
17485 ST->getOriginalAlign(), MMOFlags, AAInfo);
17488 ST->getPointerInfo().getWithOffset(4),
17489 ST->getOriginalAlign(), MMOFlags, AAInfo);
17507 ST->isUnindexed()) {
17508 EVT SVT =
Value.getOperand(0).getValueType();
17515 if (((!LegalOperations &&
ST->isSimple()) ||
17518 DAG, *
ST->getMemOperand())) {
17520 ST->getMemOperand());
17525 if (
Value.isUndef() &&
ST->isUnindexed())
17531 if (*Alignment >
ST->getAlign() &&
17532 isAligned(*Alignment,
ST->getSrcValueOffset())) {
17535 ST->getMemoryVT(), *Alignment,
17536 ST->getMemOperand()->getFlags(),
ST->getAAInfo());
17546 if (
SDValue NewST = TransformFPLoadStorePair(
N))
17553 if (
ST->isUnindexed()) {
17556 if (findBetterNeighborChains(
ST)) {
17561 Chain =
ST->getChain();
17565 if (
ST->isTruncatingStore() &&
ST->isUnindexed() &&
17566 Value.getValueType().isInteger() &&
17567 (!isa<ConstantSDNode>(
Value) ||
17568 !cast<ConstantSDNode>(
Value)->isOpaque())) {
17569 APInt TruncDemandedBits =
17571 ST->getMemoryVT().getScalarSizeInBits());
17576 AddToWorklist(
Value.getNode());
17579 ST->getMemOperand());
17599 ST->isUnindexed() &&
ST->isSimple() &&
17609 if (
StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
17610 if (
ST->isUnindexed() &&
ST->isSimple() &&
17611 ST1->isUnindexed() && ST1->isSimple()) {
17612 if (ST1->getBasePtr() == Ptr && ST1->getValue() ==
Value &&
17613 ST->getMemoryVT() == ST1->getMemoryVT()) {
17620 !ST1->getBasePtr().isUndef() &&
17623 !
ST->getMemoryVT().isScalableVector() &&
17624 !ST1->getMemoryVT().isScalableVector()) {
17627 unsigned STBitSize =
ST->getMemoryVT().getFixedSizeInBits();
17628 unsigned ChainBitSize = ST1->getMemoryVT().getFixedSizeInBits();
17633 if (STBase.
contains(DAG, STBitSize, ChainBase, ChainBitSize)) {
17634 CombineTo(ST1, ST1->getChain());
17646 ST->getMemoryVT())) {
17648 Ptr,
ST->getMemoryVT(),
ST->getMemOperand());
17659 bool Changed = mergeConsecutiveStores(
ST);
17660 if (!Changed)
break;
17669 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
17677 if (isa<ConstantFPSDNode>(
ST->getValue())) {
17678 if (
SDValue NewSt = replaceStoreOfFPConstant(
ST))
17685 return ReduceLoadOpStoreWidth(
N);
17689 const auto *LifetimeEnd = cast<LifetimeSDNode>(
N);
17690 if (!LifetimeEnd->hasOffset())
17694 LifetimeEnd->getOffset(),
false);
17698 while (!Chains.
empty()) {
17717 if (!
ST->isSimple() ||
ST->isIndexed())
17719 const TypeSize StoreSize =
ST->getMemoryVT().getStoreSize();
17727 if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
17730 dbgs() <<
"\nwithin LIFETIME_END of : ";
17731 LifetimeEndBase.dump();
dbgs() <<
"\n");
17732 CombineTo(
ST,
ST->getChain());
17775 if (!
ST->isSimple())
17808 !
Lo.getOperand(0).getValueType().isScalarInteger() ||
17809 Lo.getOperand(0).getValueSizeInBits() > HalfValBitSize ||
17811 !
Hi.getOperand(0).getValueType().isScalarInteger() ||
17812 Hi.getOperand(0).getValueSizeInBits() > HalfValBitSize)
17818 ?
Lo.getOperand(0).getValueType()
17819 :
Lo.getValueType();
17821 ?
Hi.getOperand(0).getValueType()
17822 :
Hi.getValueType();
17839 ST->getOriginalAlign(), MMOFlags, AAInfo);
17843 St0,
DL,
Hi, Ptr,
ST->getPointerInfo().getWithOffset(HalfValBitSize / 8),
17844 ST->getOriginalAlign(), MMOFlags, AAInfo);
17849 SDValue DAGCombiner::combineInsertEltToShuffle(
SDNode *
N,
unsigned InsIndex) {
17851 "Expected extract_vector_elt");
17852 SDValue InsertVal =
N->getOperand(1);
17861 isa<ConstantSDNode>(InsertVal.
getOperand(1))) {
17872 int ElementOffset = -1;
17881 while (!ArgWorkList.
empty()) {
17884 std::tie(ArgOffset, ArgVal) = ArgWorkList.
pop_back_val();
17886 if (ArgVal == InsertVal0) {
17887 ElementOffset = ArgOffset;
17893 int CurrentArgOffset =
17897 CurrentArgOffset -= Step;
17903 assert(CurrentArgOffset == ArgOffset);
17907 if (ElementOffset != -1) {
17910 auto *ExtrIndex = cast<ConstantSDNode>(InsertVal.
getOperand(1));
17911 NewMask[InsIndex] = ElementOffset + ExtrIndex->getZExtValue();
17912 assert(NewMask[InsIndex] <
17914 NewMask[InsIndex] >= 0 &&
"NewMask[InsIndex] is out of bound");
17920 return LegalShuffle;
17933 SDValue DestVec =
N->getOperand(0);
17939 if (NumSrcElts == 1)
17942 unsigned NumMaskVals = ExtendRatio * NumSrcElts;
17950 for (
unsigned i = 0; i != NumMaskVals; ++i) {
17951 if (i / NumSrcElts == InsIndex)
17952 Mask[i] = (i % NumSrcElts) + NumMaskVals;
17967 ConcatOps[0] = SubVec;
17973 AddToWorklist(PaddedSubV.
getNode());
17974 AddToWorklist(DestVecBC.
getNode());
17975 AddToWorklist(Shuf.
getNode());
17986 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
18019 unsigned Elt = IndexC->getZExtValue();
18020 if (
SDValue Shuf = combineInsertEltToShuffle(
N, Elt))
18031 && isa<ConstantSDNode>(InVec.
getOperand(2))) {
18033 if (Elt < OtherElt) {
18037 AddToWorklist(NewOp.
getNode());
18056 }
else if (InVec.
isUndef()) {
18061 assert(Ops.
size() == NumElts &&
"Unexpected vector size");
18064 if (Elt < Ops.
size()) {
18067 EVT OpVT = Ops[0].getValueType();
18075 SDValue DAGCombiner::scalarizeExtractedVectorLoad(
SDNode *EVE,
EVT InVecVT,
18093 if (NewAlign > Alignment ||
18102 Alignment = NewAlign;
18109 if (
auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
18110 int Elt = ConstEltNo->getZExtValue();
18133 if (ResultVT.
bitsGT(VecEltVT)) {
18141 OriginalLoad->
getChain(), NewPtr, MPI, VecEltVT,
18144 Chain =
Load.getValue(1);
18147 VecEltVT,
SDLoc(EVE), OriginalLoad->
getChain(), NewPtr, MPI, Alignment,
18149 Chain =
Load.getValue(1);
18150 if (ResultVT.
bitsLT(VecEltVT))
18155 WorklistRemover DeadNodes(*
this);
18160 AddToWorklist(EVE);
18163 AddToWorklistWithUsers(
Load.getNode());
18171 bool LegalOperations) {
18175 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
18206 EVT ScalarVT =
N->getValueType(0);
18240 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
18252 "BUILD_VECTOR used for scalable vectors");
18253 unsigned IndexVal =
18259 if (ScalarVT == InEltVT)
18282 unsigned ExtractIndex = IndexC->getZExtValue();
18284 unsigned BCTruncElt = IsLE ? 0 : NumElts - 1;
18295 "Extract element and scalar to vector can't change element type "
18296 "from FP to integer.");
18297 unsigned XBitWidth =
X.getValueSizeInBits();
18298 BCTruncElt = IsLE ? 0 : XBitWidth / VecEltBitWidth - 1;
18303 if (ExtractIndex == BCTruncElt && XBitWidth > VecEltBitWidth) {
18304 assert(XBitWidth % VecEltBitWidth == 0 &&
18305 "Scalar bitwidth must be a multiple of vector element bitwidth");
18321 auto *Shuf = cast<ShuffleVectorSDNode>(VecOp);
18323 int OrigElt = Shuf->getMaskElt(IndexC->getZExtValue());
18331 if (OrigElt < (
int)NumElts) {
18335 OrigElt -= NumElts;
18351 if (!LegalOperations ||
18363 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
18364 Use->getOperand(0) == VecOp &&
18365 isa<ConstantSDNode>(Use->getOperand(1));
18369 auto *CstElt = cast<ConstantSDNode>(
Use->getOperand(1));
18370 if (CstElt->getAPIntValue().ult(NumElts))
18371 DemandedElts.
setBit(CstElt->getZExtValue());
18393 bool BCNumEltsChanged =
false;
18408 BCNumEltsChanged =
true;
18414 if (!LegalOperations && !IndexC && VecOp.
hasOneUse() &&
18417 auto *
VecLoad = dyn_cast<LoadSDNode>(VecOp);
18419 return scalarizeExtractedVectorLoad(
N, VecVT,
Index,
VecLoad);
18424 if (!LegalOperations || !IndexC)
18430 int Elt = IndexC->getZExtValue();
18433 LN0 = cast<LoadSDNode>(VecOp);
18441 LN0 = cast<LoadSDNode>(VecOp.
getOperand(0));
18443 if (
auto *Shuf = dyn_cast<ShuffleVectorSDNode>(VecOp)) {
18454 if (BCNumEltsChanged)
18458 int Idx = (Elt > (int)NumElts) ? -1 : Shuf->getMaskElt(Elt);
18469 LN0 = cast<LoadSDNode>(VecOp);
18470 Elt = (Idx < (int)NumElts) ? Idx : Idx - (int)NumElts;
18490 Index.getValueType());
18508 return scalarizeExtractedVectorLoad(
N, VecVT,
Index, LN0);
18512 SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(
SDNode *
N) {
18522 unsigned NumInScalars =
N->getNumOperands();
18524 EVT VT =
N->getValueType(0);
18532 bool AllAnyExt =
true;
18534 for (
unsigned i = 0; i != NumInScalars; ++i) {
18537 if (
In.isUndef())
continue;
18543 if (!ZeroExt && !AnyExt) {
18549 EVT InTy =
In.getOperand(0).getValueType();
18555 else if (InTy != SourceType) {
18562 AllAnyExt &= AnyExt;
18569 bool ValidTypes = SourceType !=
MVT::Other &&
18585 assert(ElemRatio > 1 &&
"Invalid element size ratio");
18593 for (
unsigned i = 0,
e =
N->getNumOperands(); i !=
e; ++i) {
18597 Cast.
isUndef()) &&
"Invalid cast opcode");
18603 unsigned Index = isLE ? (i * ElemRatio) :
18604 (i * ElemRatio + (ElemRatio - 1));
18613 "Invalid vector size");
18641 EVT VT =
N->getValueType(0);
18649 unsigned NumInScalars =
N->getNumOperands();
18652 auto PeekThroughBitcast = [](
SDValue Op) {
18654 return Op.getOperand(0);
18660 for (
unsigned i = 0; i != NumInScalars; ++i) {
18661 SDValue In = PeekThroughBitcast(
N->getOperand(i));
18663 if (
In.isUndef())
continue;
18668 In = PeekThroughBitcast(
In.getOperand(0));
18679 SDValue part = PeekThroughBitcast(
In.getOperand(0));
18683 }
else if (Src != part) {
18688 SDValue ShiftAmtVal =
In.getOperand(1);
18689 if (!isa<ConstantSDNode>(ShiftAmtVal))
18692 uint64_t ShiftAmt =
In.getNode()->getConstantOperandVal(1);
18695 if (ShiftAmt != i * ScalarTypeBitsize)
18710 unsigned LeftIdx,
bool DidSplitVec) {
18713 EVT VT =
N->getValueType(0);
18718 unsigned ShuffleNumElems = NumElems;
18730 if (InVT1 != VT || InVT2 != VT) {
18731 if ((VTSize % InVT1Size == 0) && InVT1 == InVT2) {
18734 unsigned NumConcats = VTSize / InVT1Size;
18735 assert(NumConcats >= 2 &&
"Concat needs at least two inputs!");
18737 ConcatOps[0] = VecIn1;
18738 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.
getUNDEF(InVT1);
18741 }
else if (InVT1Size == VTSize * 2) {
18753 Vec2Offset = NumElems;
18754 }
else if (InVT2Size <= InVT1Size) {
18759 if (LegalOperations &&
18766 if (InVT1 != InVT2) {
18770 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
18772 ShuffleNumElems = NumElems * 2;
18779 }
else if (InVT2Size * 2 == VTSize && InVT1Size == VTSize) {
18781 ConcatOps[0] = VecIn2;
18798 for (
unsigned i = 0; i != NumElems; ++i) {
18799 if (VectorMask[i] <= 0)
18802 unsigned ExtIndex =
N->getOperand(i).getConstantOperandVal(1);
18803 if (VectorMask[i] == (
int)LeftIdx) {
18804 Mask[i] = ExtIndex;
18805 }
else if (VectorMask[i] == (
int)LeftIdx + 1) {
18806 Mask[i] = Vec2Offset + ExtIndex;
18819 if (ShuffleNumElems > NumElems)
18832 for (
int i = 0; i != NumBVOps; ++i) {
18864 if (DestSize % SrcSize != 0 ||
18870 int ZextRatio = DestSize / SrcSize;
18871 int NumMaskElts = NumBVOps * ZextRatio;
18873 for (
int i = 0; i != NumMaskElts; ++i) {
18874 if (i / ZextRatio == ZextElt) {
18879 if (i % ZextRatio == 0)
18882 ShufMask[i] = NumMaskElts;
18896 ZeroVec, ShufMask, DAG);
18907 EVT VT =
N->getValueType(0);
18920 bool UsesZeroVector =
false;
18921 unsigned NumElems =
N->getNumOperands();
18933 for (
unsigned i = 0; i != NumElems; ++i) {
18943 UsesZeroVector =
true;
18951 !isa<ConstantSDNode>(
Op.getOperand(1)))
18953 SDValue ExtractedFromVec =
Op.getOperand(0);
18958 const APInt &ExtractIdx =
Op.getConstantOperandAPInt(1);
18970 unsigned Idx = std::distance(VecIn.
begin(),
find(VecIn, ExtractedFromVec));
18971 if (Idx == VecIn.
size())
18974 VectorMask[i] = Idx;
18978 if (VecIn.
size() < 2)
18985 bool DidSplitVec =
false;
18986 if (VecIn.
size() == 2) {
18987 unsigned MaxIndex = 0;
18988 unsigned NearestPow2 = 0;
18993 for (
unsigned i = 0; i < NumElems; i++) {
18994 if (VectorMask[i] <= 0)
18996 unsigned Index =
N->getOperand(i).getConstantOperandVal(1);
18997 IndexVec[i] =
Index;
19002 if (InVT.
isSimple() && NearestPow2 > 2 && MaxIndex < NearestPow2 &&
19003 NumElems * 2 < NearestPow2) {
19004 unsigned SplitSize = NearestPow2 / 2;
19015 DidSplitVec =
true;
19017 for (
unsigned i = 0; i < NumElems; i++) {
19018 if (VectorMask[i] <= 0)
19020 VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2;
19046 for (
unsigned In = 0, Len = (VecIn.
size() / 2);
In < Len; ++
In) {
19047 unsigned LeftIdx = 2 *
In + 1;
19048 SDValue VecLeft = VecIn[LeftIdx];
19050 (LeftIdx + 1) < VecIn.
size() ? VecIn[LeftIdx + 1] :
SDValue();
19052 if (
SDValue Shuffle = createBuildVecShuffle(
DL,
N, VectorMask, VecLeft,
19053 VecRight, LeftIdx, DidSplitVec))
19061 if (UsesZeroVector)
19066 if (Shuffles.
size() == 1)
19067 return Shuffles[0];
19070 for (
int &Vec : VectorMask)
19072 Vec = Shuffles.
size() - 1;
19074 Vec = (Vec - 1) / 2;
19088 if (Shuffles.
size() % 2)
19091 for (
unsigned CurSize = Shuffles.
size(); CurSize > 1; CurSize /= 2) {
19093 Shuffles[CurSize] = DAG.
getUNDEF(VT);
19096 for (
unsigned In = 0, Len = CurSize / 2;
In < Len; ++
In) {
19100 for (
unsigned i = 0; i != NumElems; ++i) {
19101 if (VectorMask[i] == Left) {
19103 VectorMask[i] =
In;
19104 }
else if (VectorMask[i] == Right) {
19105 Mask[i] = i + NumElems;
19106 VectorMask[i] =
In;
19114 return Shuffles[0];
19122 if (LegalOperations)
19125 EVT VT =
N->getValueType(0);
19127 bool FoundZeroExtend =
false;
19129 auto checkElem = [&](
SDValue Op) -> int64_t {
19130 unsigned Opc =
Op.getOpcode();
19135 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op.getOperand(0).getOperand(1)))
19136 return C->getZExtValue();
19142 int64_t
Offset = checkElem(Op0);
19146 unsigned NumElems =
N->getNumOperands();
19148 EVT InSVT =
In.getValueType().getScalarType();
19156 for (
unsigned i = 1; i != NumElems; ++i) {
19157 if ((
Offset + i) != checkElem(
N->getOperand(i)))
19169 EVT VT =
N->getValueType(0);
19183 if (!LegalOperations) {
19186 EVT SrcVT = Splat.getValueType();
19204 assert(!V.isUndef() &&
"Splat of undef should have been handled earlier");
19209 if (!LegalTypes && (
N->getNumOperands() > 1)) {
19211 auto checkElem = [&](
SDValue Op) -> uint64_t {
19214 if (
auto CNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
19215 return CNode->getZExtValue();
19219 int Offset = checkElem(Op0);
19220 for (
unsigned i = 0; i <
N->getNumOperands(); ++i) {
19221 if (
Offset + i != checkElem(
N->getOperand(i))) {
19231 ((
Offset %
N->getValueType(0).getVectorNumElements()) ==
19237 if (
SDValue V = convertBuildVecZextToZext(
N))
19240 if (
SDValue V = reduceBuildVecExtToExtBuildVec(
N))
19243 if (
SDValue V = reduceBuildVecTruncToBitCast(
N))
19246 if (
SDValue V = reduceBuildVecToShuffle(
N))
19254 EVT OpVT =
N->getOperand(0).getValueType();
19261 EVT VT =
N->getValueType(0);
19268 bool AnyInteger =
false;
19269 bool AnyFP =
false;
19272 !
Op.getOperand(0).getValueType().isVector())
19281 EVT LastOpVT = Ops.
back().getValueType();
19298 if (
Op.getValueType() == SVT)
19318 EVT VT =
N->getValueType(0);
19319 EVT OpVT =
N->getOperand(0).getValueType();
19335 if (
Op.isUndef()) {
19336 Mask.append((
unsigned)NumOpElts, -1);
19345 int ExtIdx =
Op.getConstantOperandVal(1);
19354 Mask.append((
unsigned)NumOpElts, -1);
19365 if (0 == (NumExtElts % NumElts))
19366 ExtIdx /= (NumExtElts / NumElts);
19367 else if (0 == (NumElts % NumExtElts))
19368 ExtIdx *= (NumElts / NumExtElts);
19373 if (SV0.
isUndef() || SV0 == ExtVec) {
19375 for (
int i = 0; i != NumOpElts; ++i)
19376 Mask.push_back(i + ExtIdx);
19377 }
else if (SV1.
isUndef() || SV1 == ExtVec) {
19379 for (
int i = 0; i != NumOpElts; ++i)
19380 Mask.push_back(i + ExtIdx + NumElts);
19392 unsigned CastOpcode =
N->getOperand(0).getOpcode();
19393 switch (CastOpcode) {
19409 EVT SrcVT =
N->getOperand(0).getOperand(0).getValueType();
19417 if (
Op.getOpcode() != CastOpcode || !
Op.hasOneUse() ||
19418 Op.getOperand(0).getValueType() != SrcVT)
19426 EVT VT =
N->getValueType(0);
19431 switch (CastOpcode) {
19451 return DAG.
getNode(CastOpcode,
DL, VT, NewConcat);
19456 if (
N->getNumOperands() == 1)
19457 return N->getOperand(0);
19460 EVT VT =
N->getValueType(0);
19466 [](
const SDValue &
Op) { return Op.isUndef(); })) {
19468 assert(
In.getValueType().isVector() &&
"Must concat vectors");
19473 unsigned NumOps =
N->getNumOperands() *
In.getNumOperands();
19485 EVT SVT =
Scalar.getValueType().getVectorElementType();
19486 if (SVT ==
Scalar.getOperand(0).getValueType())
19491 if (!
Scalar.getValueType().isVector()) {
19510 if (VNTNumElms < 2)
19526 auto IsBuildVectorOrUndef = [](
const SDValue &
Op) {
19537 bool FoundMinVT =
false;
19540 EVT OpSVT =
Op.getOperand(0).getValueType();
19541 MinVT = (!FoundMinVT || OpSVT.
bitsLE(MinVT)) ? OpSVT : MinVT;
19544 assert(FoundMinVT &&
"Concat vector type mismatch");
19548 EVT OpVT =
Op.getValueType();
19557 Opnds.
append(
Op->op_begin(),
Op->op_begin() + NumElts);
19559 for (
unsigned i = 0; i != NumElts; ++i)
19567 "Concat vector type mismatch");
19590 unsigned PartNumElem =
19591 N->getOperand(0).getValueType().getVectorMinNumElements();
19593 for (
unsigned i = 0,
e =
N->getNumOperands(); i !=
e; ++i) {
19604 if (SingleSource.
getNode()) {
19605 if (
Op.getOperand(0) != SingleSource)
19608 SingleSource =
Op.getOperand(0);
19618 unsigned IdentityIndex = i * PartNumElem;
19619 if (
Op.getConstantOperandAPInt(1) != IdentityIndex)
19624 return SingleSource;
19636 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
19648 bool LegalOperations) {
19651 unsigned BinOpcode = BinOp.
getOpcode();
19657 if (VecVT != Bop0.
getValueType() || VecVT != Bop1.getValueType())
19671 if (!Sub0 || !Sub1)
19677 return DAG.
getNode(BinOpcode,
SDLoc(Extract), SubVT, Sub0, Sub1,
19684 bool LegalOperations) {
19692 auto *ExtractIndexC = dyn_cast<ConstantSDNode>(Extract->
getOperand(1));
19693 if (!ExtractIndexC)
19709 if (
C &&
C->getValueAPF().isNegZero())
19722 unsigned ExtractIndex = ExtractIndexC->getZExtValue();
19724 "Extract index is not a multiple of the vector length.");
19729 if (WideWidth % NarrowWidth != 0)
19734 unsigned NarrowingRatio = WideWidth / NarrowWidth;
19736 if (WideNumElts % NarrowingRatio != 0)
19741 WideNumElts / NarrowingRatio);
19767 if (NarrowingRatio != 2)
19782 return V.getOperand(ConcatOpNum);
19788 if (SubVecL || SubVecR) {
19819 auto *Ld = dyn_cast<LoadSDNode>(Extract->
getOperand(0));
19820 auto *ExtIdx = dyn_cast<ConstantSDNode>(Extract->
getOperand(1));
19832 unsigned Index = ExtIdx->getZExtValue();
19837 assert(
Index % NumElts == 0 &&
"The extract subvector index is not a "
19838 "multiple of the result's element count");
19857 if (
Offset.isScalable()) {
19871 EVT NVT =
N->getValueType(0);
19873 uint64_t ExtIdx =
N->getConstantOperandVal(1);
19902 if ((SrcNumElts % DestNumElts) == 0) {
19903 unsigned SrcDestRatio = SrcNumElts / DestNumElts;
19915 if ((DestNumElts % SrcNumElts) == 0) {
19916 unsigned DestSrcRatio = DestNumElts / SrcNumElts;
19921 if ((ExtIdx % DestSrcRatio) == 0) {
19923 unsigned IndexValScaled = ExtIdx / DestSrcRatio;
19950 "Concat and extract subvector do not change element type");
19951 assert((ExtIdx % ExtNumElts) == 0 &&
19952 "Extract index is not a multiple of the input vector length.");
19955 unsigned ConcatOpIdx = ExtIdx / ConcatSrcNumElts;
19960 if (ConcatSrcNumElts == ExtNumElts)
19970 unsigned NewExtIdx = ExtIdx - ConcatOpIdx * ConcatSrcNumElts;
19971 assert(NewExtIdx + ExtNumElts <= ConcatSrcNumElts &&
19972 "Trying to extract from >1 concat operand?");
19973 assert(NewExtIdx % ExtNumElts == 0 &&
19974 "Extract index is not a multiple of the input vector length.");
19989 if (ExtractSize % EltSize == 0) {
19990 unsigned NumElems = ExtractSize / EltSize;
19993 NumElems == 1 ? EltVT
20001 if (NumElems == 1) {
20010 V->
ops().slice(IdxVal, NumElems));
20020 if (!NVT.
bitsEq(SmallVT))
20064 unsigned HalfNumElts = NumElts / 2;
20067 for (
unsigned i = 0; i != NumElts; ++i) {
20070 int M =
Mask[i] < (int)NumElts ?
Mask[i] :
Mask[i] - (
int)HalfNumElts;
20071 if (i < HalfNumElts)
20074 Mask1[i - HalfNumElts] = M;
20097 EVT VT =
N->getValueType(0);
20108 unsigned NumConcats = NumElts / NumElemsPerConcat;
20110 auto IsUndefMaskElt = [](
int i) {
return i == -1; };
20115 if (NumElemsPerConcat * 2 == NumElts && N1.
isUndef() &&
20120 Mask.slice(0, NumElemsPerConcat));
20127 for (
unsigned I = 0;
I != NumConcats; ++
I) {
20128 unsigned Begin =
I * NumElemsPerConcat;
20138 for (
int i = 0; i != (int)NumElemsPerConcat; ++i) {
20139 if (IsUndefMaskElt(SubMask[i]))
20141 if ((SubMask[i] % (
int)NumElemsPerConcat) != i)
20143 int EltOpIdx = SubMask[i] / NumElemsPerConcat;
20144 if (0 <= OpIdx && EltOpIdx != OpIdx)
20148 assert(0 <= OpIdx &&
"Unknown concat_vectors op");
20202 bool IsSplat =
false;
20203 auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
20204 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
20206 if (
SDValue Splat0 = BV0->getSplatValue())
20207 IsSplat = (Splat0 == BV1->getSplatValue());
20211 for (
int M : SVN->
getMask()) {
20214 int Idx = M < (int)NumElts ? M : M - NumElts;
20215 SDValue &S = (M < (int)NumElts ? N0 : N1);
20217 Op = S.getOperand(Idx);
20231 if (!
Op.isUndef() && !isa<ConstantSDNode>(
Op) && !isa<ConstantFPSDNode>(
Op))
20232 if (!IsSplat && !DuplicateOps.
insert(
Op).second)
20243 SVT = (SVT.
bitsLT(
Op.getValueType()) ?
Op.getValueType() : SVT);
20259 bool LegalOperations) {
20273 auto isAnyExtend = [&
Mask, &NumElts](
unsigned Scale) {
20274 for (
unsigned i = 0; i != NumElts; ++i) {
20277 if ((i % Scale) == 0 &&
Mask[i] == (int)(i / Scale))
20286 for (
unsigned Scale = 2; Scale < NumElts; Scale *= 2) {
20288 if (NumElts % Scale != 0)
20290 if (!isAnyExtend(Scale))
20298 if (!LegalOperations ||
20302 SDLoc(SVN), OutVT, N0));
20337 if (ExtDstSizeInBits % ExtSrcSizeInBits != 0)
20339 unsigned ExtScale = ExtDstSizeInBits / ExtSrcSizeInBits;
20344 auto isTruncate = [&
Mask, &NumElts](
unsigned Scale) {
20345 for (
unsigned i = 0; i != NumElts; ++i) {
20348 if ((i * Scale) < NumElts &&
Mask[i] == (int)(i * Scale))
20358 if (EltSizeInBits != ExtSrcSizeInBits)
20363 if (isTruncate(ExtScale))
20378 auto *Splat = dyn_cast<ShuffleVectorSDNode>(Shuf->
getOperand(0));
20379 if (!Splat || !Splat->isSplat())
20384 assert(ShufMask.
size() == SplatMask.
size() &&
"Mask length mismatch");
20402 auto CanSimplifyToExistingSplat = [](
ArrayRef<int> UserMask,
20404 for (
unsigned i = 0,
e = UserMask.
size(); i !=
e; ++i)
20405 if (UserMask[i] != -1 && SplatMask[i] == -1 &&
20406 SplatMask[UserMask[i]] != -1)
20410 if (CanSimplifyToExistingSplat(ShufMask, SplatMask))
20416 for (
int Idx : ShufMask)
20417 NewMask.
push_back(Idx == -1 ? -1 : SplatMask[Idx]);
20420 Splat->getOperand(0), Splat->getOperand(1),
20430 auto *InnerShuf = dyn_cast<ShuffleVectorSDNode>(OuterShuf->
getOperand(0));
20431 if (!InnerShuf || !InnerShuf->getOperand(1).isUndef())
20436 unsigned NumElts = OuterMask.
size();
20437 assert(NumElts == InnerMask.
size() &&
"Mask length mismatch");
20439 int SplatIndex = -1;
20440 for (
unsigned i = 0; i != NumElts; ++i) {
20442 int OuterMaskElt = OuterMask[i];
20443 if (OuterMaskElt == -1)
20447 int InnerMaskElt = InnerMask[OuterMaskElt];
20448 if (InnerMaskElt == -1)
20452 if (SplatIndex == -1)
20453 SplatIndex = InnerMaskElt;
20456 if (SplatIndex != InnerMaskElt)
20459 CombinedMask[i] = InnerMaskElt;
20461 assert((
all_of(CombinedMask, [](
int M) {
return M == -1; }) ||
20463 "Expected a splat mask");
20467 assert(VT == InnerShuf->getValueType(0) &&
"Expected matching shuffle types");
20472 InnerShuf->getOperand(1), CombinedMask);
20480 int MaskSize =
Mask.size();
20481 int EltFromOp0 = -1;
20486 for (
int i = 0; i != MaskSize; ++i) {
20487 if (
Mask[i] >= 0 &&
Mask[i] < MaskSize) {
20489 if (EltFromOp0 != -1)
20492 }
else if (
Mask[i] != i + MaskSize) {
20512 if (ShufOp0Index == -1) {
20516 if (ShufOp0Index == -1)
20520 Mask = CommutedMask;
20529 "Shuffle mask value must be from operand 0");
20533 auto *InsIndexC = dyn_cast<ConstantSDNode>(Op0.
getOperand(2));
20534 if (!InsIndexC || InsIndexC->getSExtValue() !=
Mask[ShufOp0Index])
20558 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Shuf->
getOperand(0));
20564 for (
int i = 0,
e = (
int)
Mask.size(); i !=
e; ++i) {
20568 assert(
Mask[i] >= 0 &&
Mask[i] <
e &&
"Unexpected shuffle mask value");
20572 if (Mask0[
Mask[i]] != Mask0[i])
20581 EVT VT =
N->getValueType(0);
20598 for (
unsigned i = 0; i != NumElts; ++i) {
20600 if (Idx >= (
int)NumElts) Idx -= NumElts;
20612 bool Changed =
false;
20614 for (
unsigned i = 0; i != NumElts; ++i) {
20616 if (Idx >= (
int)NumElts) {
20670 "BUILD_VECTOR has wrong number of operands");
20672 bool AllSame =
true;
20673 for (
unsigned i = 0; i != NumElts; ++i) {
20682 for (
unsigned i = 0; i != NumElts; ++i) {
20737 int HalfNumElts = (int)NumElts / 2;
20739 for (
unsigned i = 0; i != NumElts; ++i) {
20741 if (Idx >= HalfNumElts) {
20742 assert(Idx < (
int)NumElts &&
"Shuffle mask chooses undef op");
20743 Idx -= HalfNumElts;
20775 EVT ScaleVT = SVT.
bitsLT(InnerSVT) ? VT : InnerVT;
20793 for (
int M : OuterMask)
20794 NewMask.
push_back(M < 0 ? -1 : InnerMask[M]);
20826 "Shuffle types don't match");
20830 bool HasSameOp0 = N0 == SV0;
20831 bool IsSV1Undef = SV1.
isUndef();
20832 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
20841 cast<ShuffleVectorSDNode>(N0)->isSplat() &&
20842 !cast<ShuffleVectorSDNode>(N1)->isSplat()) {
20856 if (OtherSVN->isSplat())
20862 for (
unsigned i = 0; i != NumElts; ++i) {
20866 Mask.push_back(Idx);
20871 if (Idx < (
int)NumElts) {
20874 Idx = OtherSVN->getMaskElt(Idx);
20877 Mask.push_back(Idx);
20880 CurrentVec = (Idx < (int)NumElts) ? OtherSVN->
getOperand(0)
20889 Mask.push_back(-1);
20895 Idx = Idx % NumElts;
20896 if (!SV0.getNode() || SV0 == CurrentVec) {
20900 Mask.push_back(Idx);
20903 if (!SV1.getNode() || SV1 == CurrentVec) {
20907 Mask.push_back(Idx + NumElts);
20913 if (
auto *CurrentSVN = dyn_cast<ShuffleVectorSDNode>(CurrentVec)) {
20914 int InnerIdx = CurrentSVN->getMaskElt(Idx);
20915 if (InnerIdx < 0) {
20916 Mask.push_back(-1);
20919 SDValue InnerVec = (InnerIdx < (int)NumElts)
20923 Mask.push_back(-1);
20926 InnerIdx %= NumElts;
20927 if (InnerVec == SV0) {
20928 Mask.push_back(InnerIdx);
20931 if (InnerVec == SV1) {
20932 Mask.push_back(InnerIdx + NumElts);
20956 "Shuffle types don't match");
20960 if (MergeInnerShuffle(SVN, OtherSV, N1, SV0, SV1,
Mask)) {
20989 EVT VT =
N->getValueType(0);
21016 DAG.
getUNDEF(InVecT), NewMask, DAG);
21017 if (LegalShuffle) {
21021 return LegalShuffle;
21026 InVecT.getVectorElementType(),
21029 LegalShuffle, ZeroIdx);
21040 EVT VT =
N->getValueType(0);
21044 uint64_t InsIdx =
N->getConstantOperandVal(2);
21114 if ((N0.
isUndef() || N0SrcSVT == N1SrcSVT) &&
21126 }
else if ((N1SrcSVT.
getSizeInBits() % EltSizeInBits) == 0) {
21149 if (InsIdx < OtherIdx) {
21153 AddToWorklist(NewOp.
getNode());
21167 Ops[InsIdx / Factor] = N1;
21206 unsigned Opcode =
N->getOpcode();
21227 return DAG.
getNode(NewOpcode,
SDLoc(
N),
N->getValueType(0), N0);
21240 EVT VT =
N->getValueType(0);
21247 if (LegalOperations)
21259 auto BuildClearMask = [&](
int Split) {
21260 int NumSubElts = NumElts * Split;
21264 for (
int i = 0; i != NumSubElts; ++i) {
21265 int EltIdx = i / Split;
21266 int SubIdx = i % Split;
21276 if (isa<ConstantSDNode>(Elt))
21277 Bits = cast<ConstantSDNode>(Elt)->getAPIntValue();
21278 else if (isa<ConstantFPSDNode>(Elt))
21279 Bits = cast<ConstantFPSDNode>(Elt)->getValueAPF().bitcastToAPInt();
21285 Bits =
Bits.extractBits(NumSubBits, (Split - SubIdx - 1) * NumSubBits);
21287 Bits =
Bits.extractBits(NumSubBits, SubIdx * NumSubBits);
21289 if (
Bits.isAllOnesValue())
21291 else if (
Bits == 0)
21314 for (
int Split = 1; Split <= MaxSplit; ++Split)
21316 if (
SDValue S = BuildClearMask(Split))
21327 unsigned Opcode =
N->getOpcode();
21328 EVT VT =
N->getValueType(0);
21335 int Index0, Index1;
21338 if (!Src0 || !Src1 || Index0 != Index1 ||
21359 Ops[Index0] = ScalarBO;
21370 assert(
N->getValueType(0).isVector() &&
21371 "SimplifyVBinOp only works on vectors!");
21376 EVT VT =
N->getValueType(0);
21377 unsigned Opcode =
N->getOpcode();
21395 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(LHS);
21396 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(RHS);
21397 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
21414 Shuf0->hasOneUse() && Shuf0->getOperand(1).isUndef() &&
21424 Shuf1->hasOneUse() && Shuf1->getOperand(1).isUndef() &&
21446 EVT NarrowVT =
X.getValueType();
21447 if (NarrowVT ==
Y.getValueType() &&
21449 LegalOperations)) {
21463 return Op.isUndef() ||
21464 ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
21473 if (ConcatWithConstantOrUndef(LHS) && ConcatWithConstantOrUndef(RHS) &&
21481 for (
unsigned i = 0; i != NumOperands; ++i) {
21502 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
21516 AddToWorklist(
SETCC.getNode());
21533 bool DAGCombiner::SimplifySelectOps(
SDNode *TheSelect,
SDValue LHS,
21546 CC = cast<CondCodeSDNode>(TheSelect->
getOperand(4))->get();
21553 CC = cast<CondCodeSDNode>(
Cmp.getOperand(2))->get();
21554 CmpLHS =
Cmp.getOperand(0);
21558 if (Zero && Zero->
isZero() &&
21562 CombineTo(TheSelect, Sqrt);
21605 LLD->getPointerInfo().getAddrSpace() != 0 ||
21612 LLD->getBasePtr().getValueType()))
21629 Visited.
insert(TheSelect);
21647 if ((
LLD->hasAnyUseOfValue(1) &&
21654 LLD->getBasePtr().getValueType(),
21669 if ((
LLD->hasAnyUseOfValue(1) &&
21676 LLD->getBasePtr().getValueType(),
21702 :
LLD->getExtensionType(),
21708 CombineTo(TheSelect,
Load);
21754 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
21755 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
21756 unsigned ShCt = XType.
getSizeInBits() - N2C->getAPIntValue().logBase2() - 1;
21760 AddToWorklist(Shift.
getNode());
21762 if (XType.
bitsGT(AType)) {
21764 AddToWorklist(Shift.
getNode());
21768 Shift = DAG.
getNOT(
DL, Shift, AType);
21780 AddToWorklist(Shift.
getNode());
21782 if (XType.
bitsGT(AType)) {
21784 AddToWorklist(Shift.
getNode());
21788 Shift = DAG.
getNOT(
DL, Shift, AType);
21796 EVT VT =
N->getValueType(0);
21804 EVT IntVT =
Int.getValueType();
21818 SignMask = ~SignMask;
21824 SignMask = ~SignMask;
21829 AddToWorklist(
Int.getNode());
21837 SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
21845 auto *TV = dyn_cast<ConstantFPSDNode>(N2);
21846 auto *FV = dyn_cast<ConstantFPSDNode>(N3);
21853 TLI.
isFPImmLegal(TV->getValueAPF(), TV->getValueType(0), ForCodeSize) ||
21854 TLI.
isFPImmLegal(FV->getValueAPF(), FV->getValueType(0), ForCodeSize))
21859 if (!TV->hasOneUse() && !FV->hasOneUse())
21863 const_cast<ConstantFP*
>(TV->getConstantFPValue()) };
21871 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
21880 AddToWorklist(
Cond.getNode());
21882 AddToWorklist(CstOffset.
getNode());
21884 AddToWorklist(CPIdx.
getNode());
21894 bool NotExtCompare) {
21896 if (N2 == N3)
return N2;
21901 auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode());
21902 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
21903 auto *N3C = dyn_cast<ConstantSDNode>(N3.
getNode());
21907 AddToWorklist(SCC.
getNode());
21908 if (
auto *SCCC = dyn_cast<ConstantSDNode>(SCC)) {
21911 return !(SCCC->isNullValue()) ? N2 : N3;
21916 convertSelectOfFPConstantsToLoadOffset(
DL, N0, N1, N2, N3, CC))
21919 if (
SDValue V = foldSelectCCToShiftAnd(
DL, N0, N1, N2, N3, CC))
21931 auto *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
21932 if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) {
21934 const APInt &AndMask = ConstAndRHS->getAPIntValue();
21955 bool Fold = N2C &&
isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2();
21956 bool Swap = N3C &&
isNullConstant(N2) && N3C->getAPIntValue().isPowerOf2();
21958 if ((Fold || Swap) &&
21970 if (NotExtCompare && N2C->isOne())
21976 SCC = DAG.
getSetCC(
DL, CmpResVT, N0, N1, CC);
21986 AddToWorklist(SCC.
getNode());
21987 AddToWorklist(Temp.
getNode());
21992 unsigned ShCt = N2C->getAPIntValue().logBase2();
22017 if (
auto *ValueOnZeroC = dyn_cast<ConstantSDNode>(ValueOnZero)) {
22043 bool foldBooleans) {
22045 DagCombineInfo(DAG,
Level,
false,
this);
22077 if (
C->isNullValue())
22134 EVT VT =
Op.getValueType();
22141 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
22148 AddToWorklist(Est.getNode());
22156 for (
int i = 0; i < Iterations; ++i) {
22159 if (i == Iterations - 1) {
22161 AddToWorklist(MulEst.
getNode());
22165 AddToWorklist(NewEst.
getNode());
22168 (i == Iterations - 1 ?
N : FPOne), NewEst, Flags);
22169 AddToWorklist(NewEst.
getNode());
22172 AddToWorklist(NewEst.
getNode());
22175 AddToWorklist(Est.getNode());
22180 AddToWorklist(Est.getNode());
22196 unsigned Iterations,
22198 EVT VT =
Arg.getValueType();
22208 for (
unsigned i = 0; i < Iterations; ++i) {
22228 unsigned Iterations,
22230 EVT VT =
Arg.getValueType();
22241 for (
unsigned i = 0; i < Iterations; ++i) {
22250 if (Reciprocal || (i + 1) < Iterations) {
22273 EVT VT =
Op.getValueType();
22280 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
22287 bool UseOneConstNR =
false;
22291 AddToWorklist(Est.
getNode());
22294 Est = UseOneConstNR
22295 ? buildSqrtNROneConst(
Op, Est, Iterations, Flags, Reciprocal)
22296 : buildSqrtNRTwoConst(
Op, Est, Iterations, Flags, Reciprocal);
22316 return buildSqrtEstimateImpl(
Op, Flags,
true);
22320 return buildSqrtEstimateImpl(
Op, Flags,
false);
22324 bool DAGCombiner::isAlias(
SDNode *Op0,
SDNode *Op1)
const {
22326 struct MemUseCharacteristics {
22335 auto getCharacteristics = [](
SDNode *
N) -> MemUseCharacteristics {
22336 if (
const auto *LSN = dyn_cast<LSBaseSDNode>(
N)) {
22338 if (
auto *
C = dyn_cast<ConstantSDNode>(LSN->getOffset()))
22340 ?
C->getSExtValue()
22342 ? -1 *
C->getSExtValue()
22346 return {LSN->isVolatile(), LSN->isAtomic(), LSN->getBasePtr(),
22349 LSN->getMemOperand()};
22351 if (
const auto *LN = cast<LifetimeSDNode>(
N))
22352 return {
false ,
false, LN->getOperand(1),
22353 (LN->hasOffset()) ? LN->getOffset() : 0,
22358 return {
false ,
false,
SDValue(),
22363 MemUseCharacteristics MUC0 = getCharacteristics(Op0),
22364 MUC1 = getCharacteristics(Op1);
22367 if (MUC0.BasePtr.getNode() && MUC0.BasePtr == MUC1.BasePtr &&
22368 MUC0.Offset == MUC1.Offset)
22372 if (MUC0.IsVolatile && MUC1.IsVolatile)
22377 if (MUC0.IsAtomic && MUC1.IsAtomic)
22380 if (MUC0.MMO && MUC1.MMO) {
22381 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
22382 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
22395 if (!MUC0.MMO || !MUC1.MMO)
22401 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
22402 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
22410 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
22411 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
22412 Align OrigAlignment0 = MUC0.MMO->getBaseAlign();
22413 Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
22414 auto &Size0 = MUC0.NumBytes;
22415 auto &Size1 = MUC1.NumBytes;
22416 if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
22417 Size0.hasValue() && Size1.hasValue() && *Size0 == *Size1 &&
22418 OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
22419 SrcValOffset1 % *Size1 == 0) {
22420 int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.
value();
22421 int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.
value();
22425 if ((OffAlign0 + *Size0) <= OffAlign1 || (OffAlign1 + *Size1) <= OffAlign0)
22438 if (
UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
22439 Size0.hasValue() && Size1.hasValue()) {
22441 int64_t MinOffset =
std::min(SrcValOffset0, SrcValOffset1);
22442 int64_t Overlap0 = *Size0 + SrcValOffset0 - MinOffset;
22443 int64_t Overlap1 = *Size1 + SrcValOffset1 - MinOffset;
22459 void DAGCombiner::GatherAllAliases(
SDNode *
N,
SDValue OriginalChain,
22466 const bool IsLoad = isa<LoadSDNode>(
N) && cast<LoadSDNode>(
N)->isSimple();
22470 unsigned Depth = 0;
22483 bool IsOpLoad = isa<LoadSDNode>(
C.getNode()) &&
22484 cast<LSBaseSDNode>(
C.getNode())->isSimple();
22485 if ((IsLoad && IsOpLoad) || !isAlias(
N,
C.getNode())) {
22503 if (!isAlias(
N,
C.getNode())) {
22518 while (!Chains.
empty()) {
22552 if (ImproveChain(Chain)) {
22574 GatherAllAliases(
N, OldChain, Aliases);
22577 if (Aliases.
size() == 0)
22581 if (Aliases.
size() == 1)
22590 struct UnitT { } Unit;
22591 bool operator==(
const UnitT &,
const UnitT &) {
return true; }
22592 bool operator!=(
const UnitT &,
const UnitT &) {
return false; }
22608 bool DAGCombiner::parallelizeChainedStores(
StoreSDNode *St) {
22625 if (!
BasePtr.getBase().getNode())
22629 if (
BasePtr.getBase().isUndef())
22646 if (!Chain->isSimple() || Chain->isIndexed())
22655 int64_t Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8;
22658 auto I = Intervals.find(
Offset);
22660 if (
I != Intervals.end() &&
I.start() < (
Offset + Length))
22663 if (
I != Intervals.begin() && (--
I).stop() <=
Offset)
22672 if (ChainedStores.
size() == 0)
22679 for (
unsigned I = ChainedStores.
size();
I;) {
22681 SDValue BetterChain = FindBetterChain(S, NewChain);
22685 ChainedStores[
I] = S;
22689 SDValue BetterChain = FindBetterChain(St, NewChain);
22704 auto hasImprovedChain = [&](
SDValue ST) ->
bool {
22705 return ST->getOperand(0) != NewChain;
22707 bool AddNewChain =
llvm::all_of(TFOps, hasImprovedChain);
22717 AddToWorklist(
Op.getNode());
22718 AddToWorklist(STChain);
22722 bool DAGCombiner::findBetterNeighborChains(
StoreSDNode *St) {
22729 if (!
BasePtr.getBase().getNode())
22733 if (
BasePtr.getBase().isUndef())
22737 if (parallelizeChainedStores(St))
22742 if (St->
getChain() != BetterChain) {
22743 replaceStoreChain(St, BetterChain);
22753 DAGCombiner(*
this, AA, OptLevel).Run(
Level);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static cl::opt< bool > UseAA("amdgpu-use-aa-in-codegen", cl::desc("Enable the use of AA during codegen."), cl::init(true))
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static uint64_t * getMemory(unsigned numWords)
A utility function for allocating memory and checking for allocation failure.
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI)
For the instruction sequence of store below, F and I values are bundled together as an i64 value befo...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques=false)
static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, unsigned ExtOpc, SmallVectorImpl< SDNode * > &ExtendNodes, const TargetLowering &TLI)
static bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N)
copysign(x, fp_extend(y)) -> copysign(x, y) copysign(x, fp_round(y)) -> copysign(x,...
static bool isConstantOrConstantVector(SDValue N, bool NoOpaques=false)
static cl::opt< bool > CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden, cl::desc("Enable DAG combiner's use of IR alias analysis"))
static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG)
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG)
static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift, SDValue ExtractFrom, SDValue &Mask, const SDLoc &DL)
Helper function for visitOR to extract the needed side of a rotate idiom from a shl/srl/mul/udiv.
static bool getCombineLoadStoreParts(SDNode *N, unsigned Inc, unsigned Dec, bool &IsLoad, bool &IsMasked, SDValue &Ptr, const TargetLowering &TLI)
static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG)
If a vector binop is performed on splat values, it may be profitable to extract, scalarize,...
static SDNode * getPostIndexedLoadStoreOp(SDNode *N, bool &IsLoad, bool &IsMasked, SDValue &Ptr, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, const TargetLowering &TLI)
Return true if divmod libcall is available.
static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG)
static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1, SelectionDAG &DAG, const SDLoc &DL)
Given the operands of an add/sub operation, see if the 2nd operand is a masked 0/1 whose source opera...
static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf)
If we have a unary shuffle of a shuffle, see if it can be folded away completely.
static SDValue visitFMinMax(SelectionDAG &DAG, SDNode *N, APFloat(*Op)(const APFloat &, const APFloat &))
static bool canSplitIdx(LoadSDNode *LD)
static SDValue ShrinkLoadReplaceStoreWithStore(const std::pair< unsigned, unsigned > &MaskInfo, SDValue IVal, StoreSDNode *St, DAGCombiner *DC)
Check to see if IVal is something that provides a value as specified by MaskInfo.
static ConstantSDNode * getAsNonOpaqueConstant(SDValue N)
If N is a ConstantSDNode with isOpaque() == false return it casted to a ConstantSDNode pointer else n...
static void adjustCostForPairing(SmallVectorImpl< LoadedSlice > &LoadedSlices, LoadedSlice::Cost &GlobalLSCost)
Adjust the GlobalLSCost according to the target paring capabilities and the layout of the slices.
static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift, SDValue &Mask)
Match "(X shl/srl V1) & V2" where V2 may not be present.
static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG)
static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
If we are extracting a subvector produced by a wide binary operator try to use a narrow binary operat...
static bool areUsedBitsDense(const APInt &UsedBits)
Check that all bits set in UsedBits form a dense region, i.e., UsedBits looks like 0....
static SDValue getInputChainForNode(SDNode *N)
Given a node, return its input chain if it has one, otherwise return a null sd operand.
static Optional< bool > isBigEndian(const ArrayRef< int64_t > ByteOffsets, int64_t FirstOffset)
static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG)
If we are extracting a subvector from a wide vector load, convert to a narrow load to eliminate the e...
static ElementCount numVectorEltsOrZero(EVT T)
static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG)
Given an extending node with a pop-count operand, if the target does not support a pop-count in the n...
static const Optional< ByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, bool Root=false)
Recursively traverses the expression calculating the origin of the requested byte of the given value.
static SDValue foldSelectOfConstantsUsingSra(SDNode *N, SelectionDAG &DAG)
If a (v)select has a condition value that is a sign-bit test, try to smear the condition operand sign...
static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG, SDValue X, SDValue Carry0, SDValue Carry1, SDNode *N)
If we are facing some sort of diamond carry propapagtion pattern try to break it up to generate somet...
static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
If a shuffle inserts exactly one element from a source vector operand into another vector operand and...
static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType)
static cl::opt< std::string > CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden, cl::desc("Only use DAG-combiner alias analysis in this" " function"))
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy)
static SDValue formSplatFromShuffles(ShuffleVectorSDNode *OuterShuf, SelectionDAG &DAG)
Combine shuffle of shuffle of the form: shuf (shuf X, undef, InnerMask), undef, OuterMask --> splat X...
static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG, const TargetLowering &TLI)
Return true if 'Use' is a load or a store that uses N as its base pointer and that N may be folded in...
static unsigned bigEndianByteAt(unsigned BW, unsigned i)
bool refineUniformBase(SDValue &BasePtr, SDValue &Index, SelectionDAG &DAG)
static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG, bool LegalOperations)
Transform a vector binary operation into a scalar binary operation by moving the math/logic after an ...
static cl::opt< bool > MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true), cl::desc("DAG combiner may split indexing from loads"))
static SDValue combineShuffleToVectorExtend(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static cl::opt< unsigned > StoreMergeDependenceLimit("combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10), cl::desc("Limit the number of times for the same StoreNode and RootNode " "to bail out in store merging dependence check"))
static SDValue stripTruncAndExt(SDValue Value)
static bool isContractable(SDNode *N)
static cl::opt< unsigned > TokenFactorInlineLimit("combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048), cl::desc("Limit the number of operands to inline for Token Factors"))
static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
Try to convert a wide shuffle of concatenated vectors into 2 narrow shuffles followed by concatenatio...
static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
bool refineIndexType(MaskedGatherScatterSDNode *MGS, SDValue &Index, bool Scaled, SelectionDAG &DAG)
static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG)
Try to fold a 'not' shifted sign-bit with add/sub with constant operand into a shift and add with a d...
static int getShuffleMaskIndexOfOneElementFromOp0IntoOp1(ArrayRef< int > Mask)
If the shuffle mask is taking exactly one element from the first vector operand and passing through a...
static bool areSlicesNextToEachOther(const LoadedSlice &First, const LoadedSlice &Second)
Check whether or not First and Second are next to each other in memory.
static bool isBSwapHWordPair(SDValue N, MutableArrayRef< SDNode * > Parts)
static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG)
static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS, const TargetLowering &TLI)
static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG, const TargetLowering &TLI, bool Force)
Flips a boolean if it is cheaper to compute.
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, KnownBits &Known)
static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT)
static SDValue getAsCarry(const TargetLowering &TLI, SDValue V)
static cl::opt< bool > StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden, cl::desc("Bypass the profitability model of load slicing"), cl::init(false))
Hidden option to stress test load slicing, i.e., when this option is enabled, load slicing bypasses m...
static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG)
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static std::pair< unsigned, unsigned > CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain)
Check to see if V is (and load (ptr), imm), where the load is having specific bytes cleared out.
static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset=0)
static cl::opt< bool > EnableShrinkLoadReplaceStoreWithStore("combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true), cl::desc("DAG cominber enable load/<replace bytes>/store with " "a narrower store"))
static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode CC, const TargetLowering &TLI, SelectionDAG &DAG)
Generate Min/Max node.
static SDNode * getBuildPairElt(SDNode *N, unsigned i)
static SDValue combineShiftToMULH(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize, SelectionDAG &DAG, bool IsRotate)
static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N)
OR combines for which the commuted variant will be tried as well.
static bool shouldCombineToPostInc(SDNode *N, SDValue Ptr, SDNode *PtrUse, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue combineCarryDiamond(DAGCombiner &Combiner, SelectionDAG &DAG, const TargetLowering &TLI, SDValue Carry0, SDValue Carry1, SDNode *N)
static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalTypes)
Try to fold a sext/zext/aext dag node into a ConstantSDNode or a build_vector of constants.
static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineConcatVectorOfCasts(SDNode *N, SelectionDAG &DAG)
static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG)
Try to replace shift/logic that tests if a bit is clear with mask + setcc.
static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI, SelectionDAG &DAG, SDNode *N, SDValue N0, SDValue N1, EVT VT, EVT ShiftAmountTy)
static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask)
static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG)
static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static cl::opt< bool > EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable merging multiple stores " "into a wider store"))
static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG)
static cl::opt< bool > UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true), cl::desc("Enable DAG combiner's use of TBAA"))
static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG)
static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, SelectionDAG &DAG, bool LegalOperations)
static cl::opt< bool > EnableReduceLoadOpStoreWidth("combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true), cl::desc("DAG cominber enable reducing the width of load/op/store " "sequence"))
static bool isSlicingProfitable(SmallVectorImpl< LoadedSlice > &LoadedSlices, const APInt &UsedBits, bool ForCodeSize)
Check the profitability of all involved LoadedSlice.
static unsigned littleEndianByteAt(unsigned BW, unsigned i)
static bool isBSwapHWordElement(SDValue N, MutableArrayRef< SDNode * > Parts)
Return true if the specified node is an element that makes up a 32-bit packed halfword byteswap.
static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG)
static ManagedStatic< DebugCounter > DC
Optional< std::vector< StOtherPiece > > Other
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
iv Induction Variable Users
static void removeFromWorklist(Instruction *I, std::vector< Instruction * > &Worklist)
Remove all instances of I from the worklist vector specified.
unsigned const TargetRegisterInfo * TRI
print Print MemDeps of function
This file provides utility analysis objects describing memory locations.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static cl::opt< bool > Aggressive("aggressive-ext-opt", cl::Hidden, cl::desc("Aggressive extension optimization"))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
This file describes how to lower LLVM code to machine code.
static bool hasOneUse(unsigned Reg, MachineInstr *Def, MachineRegisterInfo &MRI, MachineDominatorTree &MDT, LiveIntervals &LIS)
static constexpr int Concat[]
const fltSemantics & getSemantics() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
uint64_t getZExtValue() const
Get zero extended value.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set a given bit to 1.
APInt abs() const
Get the absolute value;.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getNullValue(unsigned numBits)
Get the '0' value.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
bool isNegative() const
Determine sign of this APInt.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
int32_t exactLogBase2() const
bool isOneValue() const
Determine if this is a value of 1.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isAllOnesValue() const
Determine if all bits are set.
bool isNullValue() const
Determine if all bits are clear.
bool getBoolValue() const
Convert APInt to a boolean value.
bool isMask(unsigned numBits) const
APInt sext(unsigned width) const
Sign extend to a new width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
APInt zextOrSelf(unsigned width) const
Zero extend or truncate to width.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
unsigned countTrailingOnes() const
Count the number of trailing one bits.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
size_t size() const
size - Get the array size.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
StringRef getValueAsString() const
Return the attribute's value as a string.
Helper struct to parse and store a memory address as base + index + offset.
bool contains(const SelectionDAG &DAG, int64_t BitSize, const BaseIndexOffset &Other, int64_t OtherBitSize, int64_t &BitOffset) const
static bool computeAliasing(const SDNode *Op0, const Optional< int64_t > NumBytes0, const SDNode *Op1, const Optional< int64_t > NumBytes1, const SelectionDAG &DAG, bool &IsAlias)
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG, int64_t &Off) const
A "pseudo-class" with methods for operating on BUILD_VECTORs.
ISD::CondCode get() const
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
bool isNegative() const
Return true if the value is negative.
const APFloat & getValueAPF() const
bool isZero() const
Return true if the value is positive or negative zero.
ConstantFP - Floating Point Values [float, double].
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX)
uint64_t getZExtValue() const
int64_t getSExtValue() const
bool isAllOnesValue() const
const APInt & getAPIntValue() const
const ConstantInt * getConstantIntValue() const
This is an important base class in LLVM.
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool isScalar() const
Counting predicates.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
AttributeList getAttributes() const
Return the attribute list for this Function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
This class is used to form a handle around another node that is persistent and is updated across invo...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
This is an important class for using LLVM in a threaded context.
Base class for LoadSDNode and StoreSDNode.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
static ElementCount getFixed(ScalarTy MinVal)
This class is used to represent ISD::LOAD nodes.
const SDValue & getOffset() const
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getIntegerVT(unsigned BitWidth)
static mvt_range all_valuetypes()
SimpleValueType Iteration.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MONonTemporal
The memory access is non-temporal.
Flags getFlags() const
Return the raw flags of the source value,.
This class is used to represent an MGATHER node.
const SDValue & getPassThru() const
ISD::LoadExtType getExtensionType() const
This is a base class used to represent MGATHER and MSCATTER nodes.
const SDValue & getScale() const
void setIndexType(ISD::MemIndexType IndexType)
bool isIndexScaled() const
const SDValue & getIndex() const
const SDValue & getBasePtr() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
const SDValue & getMask() const
This class is used to represent an MLOAD node.
bool isExpandingLoad() const
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
const SDValue & getMask() const
const SDValue & getPassThru() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
This class is used to represent an MSCATTER node.
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MSTORE node.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getBasePtr() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
const SDValue & getValue() const
const SDValue & getMask() const
const SDValue & getChain() const
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
const MachinePointerInfo & getPointerInfo() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
const SDValue & getBasePtr() const
bool isNonTemporal() const
bool isDereferenceable() const
unsigned getAlignment() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
static uint64_t getSizeOrUnknown(const TypeSize &T)
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
constexpr bool hasValue() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< value_op_iterator > op_values() const
size_t use_size() const
Return the number of uses of this node.
TypeSize getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
SDVTList getVTList() const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
const SDValue & getOperand(unsigned Num) const
ArrayRef< SDUse > ops() const
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< use_iterator > uses()
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
const SDNodeFlags getFlags() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual bool disableGenericCombines(CodeGenOpt::Level OptLevel) const
virtual bool generateFMAsInMachineCombiner(CodeGenOpt::Level OptLevel) const
Help to insert SDNodeFlags automatically in transforming.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
bool isKnownNeverZero(SDValue Op) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=None, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool shouldOptForSize() const
OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 node can overflow.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
const TargetSubtargetInfo & getSubtarget() const
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits)
See if the specified operand can be simplified with the knowledge that only the bits specified by Dem...
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
bool isKnownToBeAPowerOfTwo(SDValue Val) const
Test if the given value is known to have exactly one bit set.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0)
Test whether V has a splatted value for all the demanded elements.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
const DataLayout & getDataLayout() const
MachineFunction & getMachineFunction() const
SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, const SDNodeFlags Flags=SDNodeFlags())
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
const TargetLowering & getTargetLoweringInfo() const
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
iterator_range< allnodes_iterator > allnodes()
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
void Combine(CombineLevel Level, AAResults *AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together,...
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
const TargetLibraryInfo & getLibInfo() const
LLVMContext * getContext() const
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
const TargetMachine & getTarget() const
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
SDValue getSplatValue(SDValue V)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
A vector that has set insertion semantics.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
LLVM_NODISCARD T pop_back_val()
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
ArrayRef< int > getMask() const
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
LLVM_NODISCARD bool empty() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
LLVM_NODISCARD bool empty() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
LLVM_NODISCARD T pop_back_val()
reference emplace_back(ArgTypes &&... Args)
iterator erase(const_iterator CI)
typename SuperClass::const_iterator const_iterator
iterator insert(iterator I, T &&Elt)
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
bool has(LibFunc F) const
Tests whether a library function is available.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool enableAggressiveFMAFusion(EVT VT) const
Return true if target always beneficiates from combining into FMA for a given value type.
bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const
Returns true if be combined with to form an ISD::FMAD.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, bool *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const
Use bitwise logic to make pairs of compares more efficient.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool hasBitPreservingFPLogic(EVT VT) const
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual bool shouldRemoveExtendFromGSIndex(EVT VT) const
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT, EVT SrcVT) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT) const
Returns true if we should normalize select(N0&N1, X, Y) => select(N0, select(N1, X,...
bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const
Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes.
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to fold a pair of shifts into a mask.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
virtual bool preferIncOfAddToSubOfNot(EVT VT) const
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldKeepZExtForFP16Conv() const
Does this target require the clearing of high-order bits in a register passed to the fp16 to fp conve...
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual bool isFNegFree(EVT VT) const
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool mergeStoresAfterLegalization(EVT MemVT) const
Allow store merging for the specified type after legalization in addition to before legalization.
unsigned getGatherAllAliasesMaxDepth() const
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal or custom on this target.
virtual bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of a non-zero vector constant with the give...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
virtual bool shouldScalarizeBinop(SDValue VecOp) const
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x,...
bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool isVectorClearMaskLegal(ArrayRef< int >, EVT) const
Similar to isShuffleMaskLegal.
bool hasTargetDAGCombine(ISD::NodeType NT) const
If true, the target has custom DAG combine transformations that it can perform for the specified node...
virtual bool shouldSplatInsEltVarIndex(EVT) const
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
virtual bool isNarrowingProfitable(EVT, EVT) const
Return true if it's profitable to narrow operations of type VT1 to VT2.
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const SelectionDAG &DAG) const
Returns if it's reasonable to merge stores to MemVT size.
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool hasPairedLoad(EVT, Align &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
virtual bool convertSelectOfConstantsToMath(EVT VT) const
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const
Hooks for building estimates in place of slower divisions and square roots.
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
bool isConstFalseVal(const SDNode *N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
virtual bool IsDesirableToPromoteOp(SDValue, EVT &) const
This method query the target whether it is beneficial for dag combiner to promote the specified node.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG, const DenormalMode &Mode) const
Return a target-dependent comparison result if the input operand is suitable for use with a square ro...
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const
Return a reciprocal estimate value for the input operand.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
virtual SDValue getSqrtResultForDenormInput(SDValue Operand, SelectionDAG &DAG) const
Return a target-dependent result if the input operand is not suitable for use with a square root esti...
virtual bool getPostIndexedAddressParts(SDNode *, SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isConstTrueVal(const SDNode *N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount though its operand,...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
virtual bool getPreIndexedAddressParts(SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned NoInfsFPMath
NoInfsFPMath - This flag is enabled when the -enable-no-infs-fp-math flag is specified on the command...
unsigned NoSignedZerosFPMath
NoSignedZerosFPMath - This flag is enabled when the -enable-no-signed-zeros-fp-math is specified on t...
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine,...
ScalarTy getFixedSize() const
static TypeSize Fixed(ScalarTy MinVal)
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Type * getType() const
All values are typed, get the type of this value.
iterator_range< use_iterator > uses()
int getNumOccurrences() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ ADDCARRY
Carry-using nodes for multiple precision addition and subtraction.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ CARRY_FALSE
CARRY_FALSE - This node is used when folding other nodes, like ADDC/SUBC, which indicate the carry re...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Undef
Value of the register doesn't matter.
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
This class represents lattice values for constants.
std::string & operator+=(std::string &buffer, StringRef string)
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
bool operator<(int64_t V1, const APSInt &V2)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
STATISTIC(NumFunctions, "Total number of functions")
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool operator==(uint64_t V1, const APInt &V2)
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
bool getAlign(const Function &F, unsigned index, unsigned &align)
bool operator!=(uint64_t V1, const APInt &V2)
bool operator>=(int64_t V1, const APSInt &V2)
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 maximum semantics.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
unsigned M1(unsigned Val)
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool operator>(int64_t V1, const APSInt &V2)
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Align max(MaybeAlign Lhs, Align Rhs)
void sort(IteratorTy Start, IteratorTy End)
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
detail::ValueMatchesPoly< M > HasValue(M Matcher)
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
bool isNullOrNullSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
bool is_splat(R &&Range)
Wrapper function around std::equal to detect if all elements in a container are same.
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
unsigned M0(unsigned Val)
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
std::enable_if_t<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type > cast(const Y &Val)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
unsigned Log2(Align A)
Returns the log2 of the alignment.
bool isAllOnesOrAllOnesSplat(SDValue V)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
AliasResult
The possible results of an alias query.
@ NoAlias
The two locations do not alias at all.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 minimum semantics.
bool isOneOrOneSplat(SDValue V)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
bool operator<=(int64_t V1, const APSInt &V2)
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
opStatus
IEEE-754R 7: Default exception handling.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getIEEE()
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned getBitWidth() const
Get the bit width of this value.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
bool hasApproximateFuncs() const
bool hasAllowReciprocal() const
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
void AddToWorklist(SDNode *N)
bool recursivelyDeleteUnusedNodes(SDNode *N)
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...